def _init_events(self, events_spec: LoggedEventListSpec): event_path = self._get_event_path(kind=events_spec.kind, name=events_spec.name) # Check if the file exists otherwise initialize if not os.path.exists(event_path): check_or_create_path(event_path, is_dir=False) with open(event_path, "w") as event_file: event_file.write(events_spec.get_csv_header())
def download_file(self, url, path, **kwargs): """This function downloads a single file or several files compressed in tar.gz. If the untar args is not specified it assumes a single file. If untar is provided: False or True it appends the tar.gz extension. If untar is True: it extracts the file. If untar is False: it keeps the file compressed. """ local_path = kwargs.pop("path_to", None) local_path = local_path or get_path( settings.CLIENT_CONFIG.archive_root, self._client.run_uuid ) if path: local_path = get_path(local_path, path) _local_path = local_path untar = kwargs.get("untar") if untar is not None: _local_path = _local_path + ".tar.gz" if untar is False: local_path = _local_path check_or_create_path(_local_path, is_dir=False) if not os.path.exists(_local_path): self.download( filename=_local_path, params={"path": path}, url=url, **kwargs ) return local_path
def save_image(asset_path, image_data, height, width, colorspace): check_or_create_path(asset_path, is_dir=False) image_data.save(asset_path, format="PNG") return V1EventImage(height=height, width=width, colorspace=colorspace, path=asset_path)
def file(file_context, filepath, copy_path, track): """Create auth context.""" from polyaxon.init.file import create_file_lineage from polyaxon.utils.hashing import hash_value try: file_context = V1FileType.from_dict(ConfigSpec.read_from(file_context)) except (PolyaxonSchemaError, ValidationError) as e: Printer.print_error("received a non valid file context.") Printer.print_error("Error message: {}.".format(e)) sys.exit(1) filepath = os.path.join(filepath, file_context.filename) check_or_create_path(filepath, is_dir=False) # Clean any previous file on that path if os.path.exists(filepath): os.remove(filepath) with open(filepath, "w") as generated_file: generated_file.write(file_context.content) if file_context.chmod: subprocess.check_call(["chmod", file_context.chmod, filepath]) if copy_path: filepath = copy_file(filepath, copy_path) if track: create_file_lineage( filepath=filepath, summary={"hash": hash_value(file_context.content)}, kind=file_context.kind, ) Printer.print_success("File is initialized, path: `{}`".format(filepath))
async def sync_logs( run_uuid: str, k8s_manager: AsyncK8SManager, pod: V1Pod, last_time: Optional[AwareDT], stream: bool = False, is_running: bool = True, ): path_from = CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(run_uuid) path_from = "{}/.tmpplxlogs".format(path_from) if not is_running: delete_path(path_from) return logs, _ = await query_k8s_pod_logs( k8s_manager=k8s_manager, pod=pod, last_time=last_time, stream=stream, ) if not logs: return path_from = "{}/{}".format(path_from, pod.metadata.name) check_or_create_path(path_from, is_dir=False) async with aiofiles.open(path_from, "w") as filepath: await filepath.write(V1Logs(logs=logs).to_dict(dump=True))
async def test_download_file(): store_root = set_store() path = os.path.join(store_root, "foo") check_or_create_path(path, is_dir=True) create_tmp_files(path) await download_file(subpath="foo/0", check_cache=False) path_to = os.path.join(settings.AGENT_CONFIG.artifacts_root, "foo/0") assert os.path.exists(path_to)
def model_path(from_path: str, asset_path: str, framework: str = None, spec: Dict = None) -> V1EventModel: check_or_create_path(asset_path, is_dir=False) if os.path.isfile(from_path): shutil.copy(from_path, asset_path) else: shutil.copytree(from_path, asset_path) return V1EventModel(path=asset_path, framework=framework, spec=spec)
def copy_file_or_dir_path(from_path: str, asset_path: str, use_basename: bool = False): if use_basename: dir_name = os.path.basename(os.path.normpath(from_path)) asset_path = ( os.path.join(asset_path, dir_name) if asset_path is not None else dir_name ) check_or_create_path(asset_path, is_dir=False) if os.path.isfile(from_path): shutil.copy(from_path, asset_path) else: shutil.copytree(from_path, asset_path)
async def test_delete_file(): store_root = set_store() path = os.path.join(store_root, "foo") check_or_create_path(path, is_dir=True) create_tmp_files(path) filepath = "{}/0".format(path) assert os.path.exists(path) is True assert os.path.exists(filepath) is True await delete_file(subpath="foo/0") assert os.path.exists(path) is True assert os.path.exists(filepath) is False
async def handle_posted_data( content_file: UploadFile, root_path: str, path: str, upload: bool, is_file: bool, overwrite: bool = True, untar: bool = True, ) -> str: tmp_path = "{}/{}".format(root_path, os.path.basename( content_file.filename)).rstrip("/") if path: root_path = "{}/{}".format(root_path, path).rstrip("/") if is_file: root_path = "{}/{}".format(root_path, os.path.basename(content_file.filename)) else: if untar: root_path = "{}/{}".format(root_path, DEFAULT_UPLOADS_PATH).rstrip("/") else: root_path = tmp_path if not untar: tmp_path = root_path full_tmppath = os.path.join(settings.AGENT_CONFIG.artifacts_root, tmp_path) full_filepath = os.path.join(settings.AGENT_CONFIG.artifacts_root, root_path) if overwrite and os.path.exists(full_filepath): delete_path(full_filepath) if not overwrite and os.path.exists(full_filepath): return full_filepath # Always clean tmp path if overwrite and os.path.exists(full_tmppath): delete_path(full_tmppath) check_or_create_path(full_tmppath, is_dir=False) check_or_create_path(full_filepath, is_dir=not is_file) # Creating the new file with open(full_tmppath, "wb") as destination: for chunk in content_file.file: destination.write(chunk) if untar: untar_file(full_tmppath, extract_path=full_filepath, use_filepath=False) if upload: if is_file: await upload_file(subpath=root_path) else: await upload_dir(subpath=root_path) return root_path
def download_file(self, path_from, local_path, use_basename=True, **kwargs): local_path = os.path.abspath(local_path) if use_basename: local_path = append_basename(local_path, path_from) if local_path == path_from: return check_or_create_path(local_path, is_dir=False) if os.path.exists(path_from) and os.path.isfile(path_from): shutil.copy(path_from, local_path)
async def upload_data(subpath: str, data): path_to = get_path(settings.AGENT_CONFIG.artifacts_store.store_path, subpath) path_from = os.path.join(settings.AGENT_CONFIG.artifacts_root, subpath) check_or_create_path(path_from, is_dir=False) async with aiofiles.open(path_from, "w") as filepath_upload: await filepath_upload.write(data) manager.upload_file_or_dir( connection_type=settings.AGENT_CONFIG.artifacts_store, path_from=path_from, path_to=path_to, is_file=True, )
async def test_download_dir_archive(): store_root = set_store() path = os.path.join(store_root, "foo") check_or_create_path(path, is_dir=True) create_tmp_files(path) await download_dir(subpath="foo", to_tar=True) path_to = os.path.join(settings.AGENT_CONFIG.artifacts_root, "foo") assert os.path.exists(path_to) assert os.path.exists(path_to + "/0") assert os.path.exists(path_to + "/1") tar_path = os.path.join(settings.CLIENT_CONFIG.archive_root, "foo.tar.gz") assert os.path.exists(tar_path)
async def download_dir(subpath: str, to_tar: bool = False) -> str: path_from = get_path(settings.AGENT_CONFIG.artifacts_store.store_path, subpath) path_to = os.path.join(settings.CLIENT_CONFIG.archive_root, subpath) check_or_create_path(path_to, is_dir=True) return manager.download_file_or_dir( connection_type=settings.AGENT_CONFIG.artifacts_store, path_from=path_from, path_to=path_to, is_file=False, workers=5, to_tar=to_tar, )
def fetch_git_repo( repo_path: str, clone_url: str, revision: str, flags: List[str] = None, ): check_or_create_path(repo_path, is_dir=True) git_init(repo_path) add_remote(repo_path, clone_url) env = None if has_ssh_access(): env = {"GIT_SSH_COMMAND": get_ssh_cmd()} git_fetch(repo_path=repo_path, revision=revision, flags=flags, env=env)
async def download_file(subpath: str, check_cache=True) -> str: path_from = get_path(settings.AGENT_CONFIG.artifacts_store.store_path, subpath) path_to = os.path.join(settings.CLIENT_CONFIG.archive_root, subpath) if check_cache and os.path.exists(path_to): # file already exists return path_to check_or_create_path(path_to, is_dir=False) return manager.download_file_or_dir( connection_type=settings.AGENT_CONFIG.artifacts_store, path_from=path_from, path_to=path_to, is_file=True, )
async def download_dir(subpath: str, to_tar: bool = False) -> Optional[str]: path_from = get_path(settings.AGENT_CONFIG.artifacts_store.store_path, subpath) path_to = os.path.join(settings.CLIENT_CONFIG.archive_root, subpath) check_or_create_path(path_to, is_dir=True) try: return manager.download_file_or_dir( connection_type=settings.AGENT_CONFIG.artifacts_store, path_from=path_from, path_to=path_to, is_file=False, workers=5, to_tar=to_tar, ) except (OSError, PolyaxonException) as e: logger.warning("Could not download %s. Error %s" % (path_from, e)) return None
def download_file(self, url, path, **kwargs): local_path = get_path( settings.CLIENT_CONFIG.archive_root, self._client.run_uuid, ) _local_path = local_path if path: _local_path = get_path(local_path, path) if kwargs.get("untar"): _local_path = _local_path + ".tar.gz" check_or_create_path(_local_path, is_dir=False) if not os.path.exists(_local_path): self.download(filename=_local_path, params={"path": path}, url=url, **kwargs) return local_path
def __init__(self, run_path: str, max_queue_size: int = 20, flush_secs: int = 10): """Creates a `ResourceFileWriter`. Args: run_path: A string. Directory where events files will be written. max_queue_size: Integer. Size of the queue for pending events and summaries. flush_secs: Number. How often, in seconds, to flush the pending events and summaries to disk. """ super().__init__(run_path=run_path) check_or_create_path(get_resource_path(run_path), is_dir=True) self._async_writer = ResourceAsyncManager( EventWriter(self._run_path, backend=EventWriter.RESOURCES_BACKEND), max_queue_size, flush_secs, )
def make_video(asset_path: str, tensor, fps, content_type="gif", asset_rel_path: str = None): try: import moviepy # noqa: F401 except ImportError: logger.warning(MOVIEPY_ERROR_MESSAGE) return UNKNOWN try: from moviepy import editor as mpy except ImportError: logger.warning( "moviepy is installed, but can't import moviepy.editor.", "Some packages could be missing [imageio, requests]", ) return t, h, w, c = tensor.shape # encode sequence of images into gif string clip = mpy.ImageSequenceClip(list(tensor), fps=fps) check_or_create_path(asset_path, is_dir=False) try: # older version of moviepy if content_type == "gif": clip.write_gif(asset_path, verbose=False, progress_bar=False) else: clip.write_videofile(asset_path, verbose=False, progress_bar=False) except TypeError: if content_type == "gif": clip.write_gif(asset_path, verbose=False) else: clip.write_videofile(asset_path, verbose=False) return V1EventVideo( height=h, width=w, colorspace=c, path=asset_rel_path or asset_path, content_type=content_type, )
def upload_dir( self, dirname, path_to, use_basename=True, workers=0, last_time=None, exclude: List[str] = None, ): if use_basename: path_to = append_basename(path_to, dirname) if dirname == path_to: return check_or_create_path(path_to, is_dir=True) pool, future_results = self.init_pool(workers) # Turn the path to absolute paths dirname = os.path.abspath(dirname) with get_files_in_path_context(dirname, exclude=exclude) as files: for f in files: # If last time is provided we check if we should re-upload the file if last_time and not file_modified_since( filepath=f, last_time=last_time ): continue file_blob = os.path.join(path_to, os.path.relpath(f, dirname)) future_results = self.submit_pool( workers=workers, pool=pool, future_results=future_results, fn=self.upload_file, filename=f, path_to=file_blob, use_basename=False, ) if workers: futures.wait(future_results) self.close_pool(pool=pool)
async def download_file(subpath: str, check_cache=True) -> Optional[str]: path_from = get_path(settings.AGENT_CONFIG.artifacts_store.store_path, subpath) path_to = os.path.join(settings.CLIENT_CONFIG.archive_root, subpath) if os.path.exists(path_to): if check_cache: # file already exists return path_to else: os.remove(path_to) check_or_create_path(path_to, is_dir=False) try: return manager.download_file_or_dir( connection_type=settings.AGENT_CONFIG.artifacts_store, path_from=path_from, path_to=path_to, is_file=True, ) except PolyaxonException: return None
def audio(asset_path: str, tensor, sample_rate=44100, asset_rel_path: str = None): if not np: logger.warning(NUMPY_ERROR_MESSAGE) return UNKNOWN tensor = to_np(tensor) tensor = tensor.squeeze() if abs(tensor).max() > 1: print("warning: audio amplitude out of range, auto clipped.") tensor = tensor.clip(-1, 1) assert tensor.ndim == 1, "input tensor should be 1 dimensional." tensor_list = [int(32767.0 * x) for x in tensor] import wave import struct check_or_create_path(asset_path, is_dir=False) wave_write = wave.open(asset_path, "wb") wave_write.setnchannels(1) wave_write.setsampwidth(2) wave_write.setframerate(sample_rate) tensor_enc = b"" for v in tensor_list: tensor_enc += struct.pack("<h", v) wave_write.writeframes(tensor_enc) wave_write.close() return V1EventAudio( sample_rate=sample_rate, num_channels=1, length_frames=len(tensor_list), path=asset_rel_path or asset_path, content_type="audio/wav", )
def image_path(from_path: str, asset_path: str) -> V1EventImage: check_or_create_path(asset_path, is_dir=False) shutil.copy(from_path, asset_path) return V1EventImage(path=asset_path)
def artifact_path(from_path: str, asset_path: str, kind: str) -> V1EventArtifact: check_or_create_path(asset_path, is_dir=False) shutil.copy(from_path, asset_path) return V1EventArtifact(kind=kind, path=asset_path)
def dataframe_path(from_path: str, asset_path: str, content_type: str = None) -> V1EventDataframe: check_or_create_path(asset_path, is_dir=False) shutil.copy(from_path, asset_path) return V1EventDataframe(path=asset_path, content_type=content_type)
def copy_dir_path(from_path: str, asset_path: str): check_or_create_path(asset_path, is_dir=False) shutil.copytree(from_path, asset_path)
def __init__(self, run_path: str): self._run_path = run_path check_or_create_path(run_path, is_dir=True)
def audio_path(from_path: str, asset_path: str, content_type=None) -> V1EventAudio: check_or_create_path(asset_path, is_dir=False) shutil.copy(from_path, asset_path) return V1EventAudio(path=asset_path, content_type=content_type)
def copy_file_or_dir_path(from_path: str, asset_path: str): check_or_create_path(asset_path, is_dir=False) if os.path.isfile(from_path): shutil.copy(from_path, asset_path) else: shutil.copytree(from_path, asset_path)