def resolve_yaml(self, info):
import ruamel.yaml
if ruamel.yaml.version_info < (0, 15):
yaml = ruamel.yaml
load_fn = yaml.safe_load
else:
from ruamel.yaml import YAML
yaml = YAML()
yaml.explict_start = True
load_fn = yaml.load
from ml_dash.config import Args
with open(join(Args.logdir, self.id[1:]), "r") as f:
return load_fn('\n'.join(f))
def log(self, key, data, dtype, options: LogOptions = None):
"""
handler function for writing data to the server. Can be called directly.
:param key:
:param data:
:param dtype:
:param options:
:return:
"""
# todo: overwrite mode is not tested and not in-use.
write_mode = "w" if options and options.overwrite else "a"
if dtype == "log":
abs_path = os.path.join(self.data_dir, key)
try:
with open(abs_path, write_mode + 'b') as f:
dill.dump(data, f)
except FileNotFoundError:
os.makedirs(os.path.dirname(abs_path))
with open(abs_path, write_mode + 'b') as f:
dill.dump(data, f)
if dtype == "byte":
abs_path = os.path.join(self.data_dir, key)
try:
with open(abs_path, write_mode + 'b') as f:
f.write(data)
except FileNotFoundError:
os.makedirs(os.path.dirname(abs_path))
with open(abs_path, write_mode + 'b') as f:
f.write(data)
elif dtype.startswith("text"):
abs_path = os.path.join(self.data_dir, key)
try:
with open(abs_path, write_mode + "+") as f:
f.write(data)
except FileNotFoundError:
os.makedirs(os.path.dirname(abs_path))
with open(abs_path, write_mode + "+") as f:
f.write(data)
elif dtype.startswith("yaml"):
import ruamel.yaml
if ruamel.yaml.version_info < (0, 15):
yaml = ruamel.yaml
StringIO = ruamel.yaml.StringIO
load_fn = yaml.safe_load
else:
from ruamel.yaml import YAML, StringIO
yaml = YAML()
yaml.explict_start = True
load_fn = yaml.load
stream = StringIO()
yaml.dump(data, stream)
output = stream.getvalue()
abs_path = os.path.join(self.data_dir, key)
try:
with open(abs_path, write_mode + "+") as f:
if options.write_mode == 'key':
d = load_fn('\n'.join(f))
if d is not None:
d.update(output)
output = d
f.write(output)
except FileNotFoundError:
os.makedirs(os.path.dirname(abs_path))
with open(abs_path, write_mode + "+") as f:
if options.write_mode == 'key':
d = load_fn('\n'.join(f))
if d is not None:
d.update(output)
output = d
f.write(output)
elif dtype.startswith("image"):
abs_path = os.path.join(self.data_dir, key)
if "." not in key:
abs_path = abs_path + ".png"
from PIL import Image
assert data.dtype in ALLOWED_TYPES, "image datatype must be one of {}".format(
ALLOWED_TYPES)
if len(data.shape) == 3 and data.shape[-1] == 1:
data.resize(data.shape[:-1])
im = Image.fromarray(data)
try:
im.save(abs_path)
except FileNotFoundError:
os.makedirs(os.path.dirname(abs_path))
im.save(abs_path)
