def _init_events(self, events_spec: LoggedEventListSpec):
event_path = self._get_event_path(kind=events_spec.kind, name=events_spec.name)
# Check if the file exists otherwise initialize
if not os.path.exists(event_path):
check_or_create_path(event_path, is_dir=False)
with open(event_path, "w") as event_file:
event_file.write(events_spec.get_csv_header())
def download_file(self, url, path, **kwargs):
"""This function downloads a single file or several files compressed in tar.gz.
If the untar args is not specified it assumes a single file.
If untar is provided: False or True it appends the tar.gz extension.
If untar is True: it extracts the file.
If untar is False: it keeps the file compressed.
"""
local_path = kwargs.pop("path_to", None)
local_path = local_path or get_path(
settings.CLIENT_CONFIG.archive_root, self._client.run_uuid
)
if path:
local_path = get_path(local_path, path)
_local_path = local_path
untar = kwargs.get("untar")
if untar is not None:
_local_path = _local_path + ".tar.gz"
if untar is False:
local_path = _local_path
check_or_create_path(_local_path, is_dir=False)
if not os.path.exists(_local_path):
self.download(
filename=_local_path, params={"path": path}, url=url, **kwargs
)
return local_path
def save_image(asset_path, image_data, height, width, colorspace):
check_or_create_path(asset_path, is_dir=False)
image_data.save(asset_path, format="PNG")
return V1EventImage(height=height,
width=width,
colorspace=colorspace,
path=asset_path)
def file(file_context, filepath, copy_path, track):
"""Create auth context."""
from polyaxon.init.file import create_file_lineage
from polyaxon.utils.hashing import hash_value
try:
file_context = V1FileType.from_dict(ConfigSpec.read_from(file_context))
except (PolyaxonSchemaError, ValidationError) as e:
Printer.print_error("received a non valid file context.")
Printer.print_error("Error message: {}.".format(e))
sys.exit(1)
filepath = os.path.join(filepath, file_context.filename)
check_or_create_path(filepath, is_dir=False)
# Clean any previous file on that path
if os.path.exists(filepath):
os.remove(filepath)
with open(filepath, "w") as generated_file:
generated_file.write(file_context.content)
if file_context.chmod:
subprocess.check_call(["chmod", file_context.chmod, filepath])
if copy_path:
filepath = copy_file(filepath, copy_path)
if track:
create_file_lineage(
filepath=filepath,
summary={"hash": hash_value(file_context.content)},
kind=file_context.kind,
)
Printer.print_success("File is initialized, path: `{}`".format(filepath))
async def sync_logs(
run_uuid: str,
k8s_manager: AsyncK8SManager,
pod: V1Pod,
last_time: Optional[AwareDT],
stream: bool = False,
is_running: bool = True,
):
path_from = CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(run_uuid)
path_from = "{}/.tmpplxlogs".format(path_from)
if not is_running:
delete_path(path_from)
return
logs, _ = await query_k8s_pod_logs(
k8s_manager=k8s_manager,
pod=pod,
last_time=last_time,
stream=stream,
)
if not logs:
return
path_from = "{}/{}".format(path_from, pod.metadata.name)
check_or_create_path(path_from, is_dir=False)
async with aiofiles.open(path_from, "w") as filepath:
await filepath.write(V1Logs(logs=logs).to_dict(dump=True))
async def test_download_file():
store_root = set_store()
path = os.path.join(store_root, "foo")
check_or_create_path(path, is_dir=True)
create_tmp_files(path)
await download_file(subpath="foo/0", check_cache=False)
path_to = os.path.join(settings.AGENT_CONFIG.artifacts_root, "foo/0")
assert os.path.exists(path_to)
def model_path(from_path: str,
asset_path: str,
framework: str = None,
spec: Dict = None) -> V1EventModel:
check_or_create_path(asset_path, is_dir=False)
if os.path.isfile(from_path):
shutil.copy(from_path, asset_path)
else:
shutil.copytree(from_path, asset_path)
return V1EventModel(path=asset_path, framework=framework, spec=spec)
def copy_file_or_dir_path(from_path: str, asset_path: str, use_basename: bool = False):
if use_basename:
dir_name = os.path.basename(os.path.normpath(from_path))
asset_path = (
os.path.join(asset_path, dir_name) if asset_path is not None else dir_name
)
check_or_create_path(asset_path, is_dir=False)
if os.path.isfile(from_path):
shutil.copy(from_path, asset_path)
else:
shutil.copytree(from_path, asset_path)
async def test_delete_file():
store_root = set_store()
path = os.path.join(store_root, "foo")
check_or_create_path(path, is_dir=True)
create_tmp_files(path)
filepath = "{}/0".format(path)
assert os.path.exists(path) is True
assert os.path.exists(filepath) is True
await delete_file(subpath="foo/0")
assert os.path.exists(path) is True
assert os.path.exists(filepath) is False
async def handle_posted_data(
content_file: UploadFile,
root_path: str,
path: str,
upload: bool,
is_file: bool,
overwrite: bool = True,
untar: bool = True,
) -> str:
tmp_path = "{}/{}".format(root_path, os.path.basename(
content_file.filename)).rstrip("/")
if path:
root_path = "{}/{}".format(root_path, path).rstrip("/")
if is_file:
root_path = "{}/{}".format(root_path,
os.path.basename(content_file.filename))
else:
if untar:
root_path = "{}/{}".format(root_path,
DEFAULT_UPLOADS_PATH).rstrip("/")
else:
root_path = tmp_path
if not untar:
tmp_path = root_path
full_tmppath = os.path.join(settings.AGENT_CONFIG.artifacts_root, tmp_path)
full_filepath = os.path.join(settings.AGENT_CONFIG.artifacts_root,
root_path)
if overwrite and os.path.exists(full_filepath):
delete_path(full_filepath)
if not overwrite and os.path.exists(full_filepath):
return full_filepath
# Always clean tmp path
if overwrite and os.path.exists(full_tmppath):
delete_path(full_tmppath)
check_or_create_path(full_tmppath, is_dir=False)
check_or_create_path(full_filepath, is_dir=not is_file)
# Creating the new file
with open(full_tmppath, "wb") as destination:
for chunk in content_file.file:
destination.write(chunk)
if untar:
untar_file(full_tmppath,
extract_path=full_filepath,
use_filepath=False)
if upload:
if is_file:
await upload_file(subpath=root_path)
else:
await upload_dir(subpath=root_path)
return root_path
def download_file(self, path_from, local_path, use_basename=True, **kwargs):
local_path = os.path.abspath(local_path)
if use_basename:
local_path = append_basename(local_path, path_from)
if local_path == path_from:
return
check_or_create_path(local_path, is_dir=False)
if os.path.exists(path_from) and os.path.isfile(path_from):
shutil.copy(path_from, local_path)
async def upload_data(subpath: str, data):
path_to = get_path(settings.AGENT_CONFIG.artifacts_store.store_path,
subpath)
path_from = os.path.join(settings.AGENT_CONFIG.artifacts_root, subpath)
check_or_create_path(path_from, is_dir=False)
async with aiofiles.open(path_from, "w") as filepath_upload:
await filepath_upload.write(data)
manager.upload_file_or_dir(
connection_type=settings.AGENT_CONFIG.artifacts_store,
path_from=path_from,
path_to=path_to,
is_file=True,
)
async def test_download_dir_archive():
store_root = set_store()
path = os.path.join(store_root, "foo")
check_or_create_path(path, is_dir=True)
create_tmp_files(path)
await download_dir(subpath="foo", to_tar=True)
path_to = os.path.join(settings.AGENT_CONFIG.artifacts_root, "foo")
assert os.path.exists(path_to)
assert os.path.exists(path_to + "/0")
assert os.path.exists(path_to + "/1")
tar_path = os.path.join(settings.CLIENT_CONFIG.archive_root, "foo.tar.gz")
assert os.path.exists(tar_path)
async def download_dir(subpath: str, to_tar: bool = False) -> str:
path_from = get_path(settings.AGENT_CONFIG.artifacts_store.store_path,
subpath)
path_to = os.path.join(settings.CLIENT_CONFIG.archive_root, subpath)
check_or_create_path(path_to, is_dir=True)
return manager.download_file_or_dir(
connection_type=settings.AGENT_CONFIG.artifacts_store,
path_from=path_from,
path_to=path_to,
is_file=False,
workers=5,
to_tar=to_tar,
)
def fetch_git_repo(
repo_path: str,
clone_url: str,
revision: str,
flags: List[str] = None,
):
check_or_create_path(repo_path, is_dir=True)
git_init(repo_path)
add_remote(repo_path, clone_url)
env = None
if has_ssh_access():
env = {"GIT_SSH_COMMAND": get_ssh_cmd()}
git_fetch(repo_path=repo_path, revision=revision, flags=flags, env=env)
async def download_file(subpath: str, check_cache=True) -> str:
path_from = get_path(settings.AGENT_CONFIG.artifacts_store.store_path,
subpath)
path_to = os.path.join(settings.CLIENT_CONFIG.archive_root, subpath)
if check_cache and os.path.exists(path_to):
# file already exists
return path_to
check_or_create_path(path_to, is_dir=False)
return manager.download_file_or_dir(
connection_type=settings.AGENT_CONFIG.artifacts_store,
path_from=path_from,
path_to=path_to,
is_file=True,
)
async def download_dir(subpath: str, to_tar: bool = False) -> Optional[str]:
path_from = get_path(settings.AGENT_CONFIG.artifacts_store.store_path,
subpath)
path_to = os.path.join(settings.CLIENT_CONFIG.archive_root, subpath)
check_or_create_path(path_to, is_dir=True)
try:
return manager.download_file_or_dir(
connection_type=settings.AGENT_CONFIG.artifacts_store,
path_from=path_from,
path_to=path_to,
is_file=False,
workers=5,
to_tar=to_tar,
)
except (OSError, PolyaxonException) as e:
logger.warning("Could not download %s. Error %s" % (path_from, e))
return None
def download_file(self, url, path, **kwargs):
local_path = get_path(
settings.CLIENT_CONFIG.archive_root,
self._client.run_uuid,
)
_local_path = local_path
if path:
_local_path = get_path(local_path, path)
if kwargs.get("untar"):
_local_path = _local_path + ".tar.gz"
check_or_create_path(_local_path, is_dir=False)
if not os.path.exists(_local_path):
self.download(filename=_local_path,
params={"path": path},
url=url,
**kwargs)
return local_path
def __init__(self, run_path: str, max_queue_size: int = 20, flush_secs: int = 10):
"""Creates a `ResourceFileWriter`.
Args:
run_path: A string. Directory where events files will be written.
max_queue_size: Integer. Size of the queue for pending events and summaries.
flush_secs: Number. How often, in seconds, to flush the
pending events and summaries to disk.
"""
super().__init__(run_path=run_path)
check_or_create_path(get_resource_path(run_path), is_dir=True)
self._async_writer = ResourceAsyncManager(
EventWriter(self._run_path, backend=EventWriter.RESOURCES_BACKEND),
max_queue_size,
flush_secs,
)
def make_video(asset_path: str,
tensor,
fps,
content_type="gif",
asset_rel_path: str = None):
try:
import moviepy # noqa: F401
except ImportError:
logger.warning(MOVIEPY_ERROR_MESSAGE)
return UNKNOWN
try:
from moviepy import editor as mpy
except ImportError:
logger.warning(
"moviepy is installed, but can't import moviepy.editor.",
"Some packages could be missing [imageio, requests]",
)
return
t, h, w, c = tensor.shape
# encode sequence of images into gif string
clip = mpy.ImageSequenceClip(list(tensor), fps=fps)
check_or_create_path(asset_path, is_dir=False)
try: # older version of moviepy
if content_type == "gif":
clip.write_gif(asset_path, verbose=False, progress_bar=False)
else:
clip.write_videofile(asset_path, verbose=False, progress_bar=False)
except TypeError:
if content_type == "gif":
clip.write_gif(asset_path, verbose=False)
else:
clip.write_videofile(asset_path, verbose=False)
return V1EventVideo(
height=h,
width=w,
colorspace=c,
path=asset_rel_path or asset_path,
content_type=content_type,
)
def upload_dir(
self,
dirname,
path_to,
use_basename=True,
workers=0,
last_time=None,
exclude: List[str] = None,
):
if use_basename:
path_to = append_basename(path_to, dirname)
if dirname == path_to:
return
check_or_create_path(path_to, is_dir=True)
pool, future_results = self.init_pool(workers)
# Turn the path to absolute paths
dirname = os.path.abspath(dirname)
with get_files_in_path_context(dirname, exclude=exclude) as files:
for f in files:
# If last time is provided we check if we should re-upload the file
if last_time and not file_modified_since(
filepath=f, last_time=last_time
):
continue
file_blob = os.path.join(path_to, os.path.relpath(f, dirname))
future_results = self.submit_pool(
workers=workers,
pool=pool,
future_results=future_results,
fn=self.upload_file,
filename=f,
path_to=file_blob,
use_basename=False,
)
if workers:
futures.wait(future_results)
self.close_pool(pool=pool)
async def download_file(subpath: str, check_cache=True) -> Optional[str]:
path_from = get_path(settings.AGENT_CONFIG.artifacts_store.store_path,
subpath)
path_to = os.path.join(settings.CLIENT_CONFIG.archive_root, subpath)
if os.path.exists(path_to):
if check_cache:
# file already exists
return path_to
else:
os.remove(path_to)
check_or_create_path(path_to, is_dir=False)
try:
return manager.download_file_or_dir(
connection_type=settings.AGENT_CONFIG.artifacts_store,
path_from=path_from,
path_to=path_to,
is_file=True,
)
except PolyaxonException:
return None
def audio(asset_path: str,
tensor,
sample_rate=44100,
asset_rel_path: str = None):
if not np:
logger.warning(NUMPY_ERROR_MESSAGE)
return UNKNOWN
tensor = to_np(tensor)
tensor = tensor.squeeze()
if abs(tensor).max() > 1:
print("warning: audio amplitude out of range, auto clipped.")
tensor = tensor.clip(-1, 1)
assert tensor.ndim == 1, "input tensor should be 1 dimensional."
tensor_list = [int(32767.0 * x) for x in tensor]
import wave
import struct
check_or_create_path(asset_path, is_dir=False)
wave_write = wave.open(asset_path, "wb")
wave_write.setnchannels(1)
wave_write.setsampwidth(2)
wave_write.setframerate(sample_rate)
tensor_enc = b""
for v in tensor_list:
tensor_enc += struct.pack("<h", v)
wave_write.writeframes(tensor_enc)
wave_write.close()
return V1EventAudio(
sample_rate=sample_rate,
num_channels=1,
length_frames=len(tensor_list),
path=asset_rel_path or asset_path,
content_type="audio/wav",
)
def image_path(from_path: str, asset_path: str) -> V1EventImage:
check_or_create_path(asset_path, is_dir=False)
shutil.copy(from_path, asset_path)
return V1EventImage(path=asset_path)
def artifact_path(from_path: str, asset_path: str,
kind: str) -> V1EventArtifact:
check_or_create_path(asset_path, is_dir=False)
shutil.copy(from_path, asset_path)
return V1EventArtifact(kind=kind, path=asset_path)
def dataframe_path(from_path: str,
asset_path: str,
content_type: str = None) -> V1EventDataframe:
check_or_create_path(asset_path, is_dir=False)
shutil.copy(from_path, asset_path)
return V1EventDataframe(path=asset_path, content_type=content_type)
def copy_dir_path(from_path: str, asset_path: str):
check_or_create_path(asset_path, is_dir=False)
shutil.copytree(from_path, asset_path)
def __init__(self, run_path: str):
self._run_path = run_path
check_or_create_path(run_path, is_dir=True)
def audio_path(from_path: str,
asset_path: str,
content_type=None) -> V1EventAudio:
check_or_create_path(asset_path, is_dir=False)
shutil.copy(from_path, asset_path)
return V1EventAudio(path=asset_path, content_type=content_type)
def copy_file_or_dir_path(from_path: str, asset_path: str):
check_or_create_path(asset_path, is_dir=False)
if os.path.isfile(from_path):
shutil.copy(from_path, asset_path)
else:
shutil.copytree(from_path, asset_path)
