def resolve_yaml(self, info): import ruamel.yaml if ruamel.yaml.version_info < (0, 15): yaml = ruamel.yaml load_fn = yaml.safe_load else: from ruamel.yaml import YAML yaml = YAML() yaml.explict_start = True load_fn = yaml.load from ml_dash.config import Args with open(join(Args.logdir, self.id[1:]), "r") as f: return load_fn('\n'.join(f))
def log(self, key, data, dtype, options: LogOptions = None): """ handler function for writing data to the server. Can be called directly. :param key: :param data: :param dtype: :param options: :return: """ # todo: overwrite mode is not tested and not in-use. write_mode = "w" if options and options.overwrite else "a" if dtype == "log": abs_path = os.path.join(self.data_dir, key) try: with open(abs_path, write_mode + 'b') as f: dill.dump(data, f) except FileNotFoundError: os.makedirs(os.path.dirname(abs_path)) with open(abs_path, write_mode + 'b') as f: dill.dump(data, f) if dtype == "byte": abs_path = os.path.join(self.data_dir, key) try: with open(abs_path, write_mode + 'b') as f: f.write(data) except FileNotFoundError: os.makedirs(os.path.dirname(abs_path)) with open(abs_path, write_mode + 'b') as f: f.write(data) elif dtype.startswith("text"): abs_path = os.path.join(self.data_dir, key) try: with open(abs_path, write_mode + "+") as f: f.write(data) except FileNotFoundError: os.makedirs(os.path.dirname(abs_path)) with open(abs_path, write_mode + "+") as f: f.write(data) elif dtype.startswith("yaml"): import ruamel.yaml if ruamel.yaml.version_info < (0, 15): yaml = ruamel.yaml StringIO = ruamel.yaml.StringIO load_fn = yaml.safe_load else: from ruamel.yaml import YAML, StringIO yaml = YAML() yaml.explict_start = True load_fn = yaml.load stream = StringIO() yaml.dump(data, stream) output = stream.getvalue() abs_path = os.path.join(self.data_dir, key) try: with open(abs_path, write_mode + "+") as f: if options.write_mode == 'key': d = load_fn('\n'.join(f)) if d is not None: d.update(output) output = d f.write(output) except FileNotFoundError: os.makedirs(os.path.dirname(abs_path)) with open(abs_path, write_mode + "+") as f: if options.write_mode == 'key': d = load_fn('\n'.join(f)) if d is not None: d.update(output) output = d f.write(output) elif dtype.startswith("image"): abs_path = os.path.join(self.data_dir, key) if "." not in key: abs_path = abs_path + ".png" from PIL import Image assert data.dtype in ALLOWED_TYPES, "image datatype must be one of {}".format( ALLOWED_TYPES) if len(data.shape) == 3 and data.shape[-1] == 1: data.resize(data.shape[:-1]) im = Image.fromarray(data) try: im.save(abs_path) except FileNotFoundError: os.makedirs(os.path.dirname(abs_path)) im.save(abs_path)