def find_files_by_query(cwd, query="**/*.*", **kwargs):
from ml_dash.config import Args
assert isabs(cwd), "the current work directory need to be an absolute path."
_cwd = realpath(join(Args.logdir, cwd[1:])).rstrip('/')
parameter_files = find_files(_cwd, query, **kwargs)
return [
# note: not sure about the name.
File(id=join(cwd.rstrip('/'), p['path']),
name=basename(p['path']),
path=join(cwd.rstrip('/'), p['path']))
for p in parameter_files
]
def find_experiments(cwd, **kwargs):
from ml_dash.config import Args
assert isabs(cwd), "the current work directory need to be an absolute path."
_cwd = realpath(join(Args.logdir, cwd[1:])).rstrip('/')
parameter_files = find_files(_cwd, "**/parameters.pkl", **kwargs)
return [
# note: not sure about the name.
Experiment(id=join(cwd.rstrip('/'), p['dir']),
name=basename(p['dir']) or ".",
path=join(cwd.rstrip('/'), p['dir']),
parameters=join(cwd.rstrip('/'), p['path']), )
for p in parameter_files
]
def find_metrics(cwd, **kwargs):
from ml_dash.config import Args
_cwd = realpath(join(Args.logdir, cwd[1:]))
parameter_files = find_files(_cwd, "**/metrics.pkl", **kwargs)
for p in parameter_files:
yield Metrics(id=join(cwd, p['path']), name="metrics.pkl")
def find_parameters(cwd, **kwargs):
from ml_dash.config import Args
_cwd = realpath(pJoin(Args.logdir, cwd[1:]))
parameter_files = find_files(_cwd, "parameters.pkl", **kwargs)
for p in parameter_files:
yield Parameters(id=pJoin(cwd, p['path']))
def typify(obj):
t_str, obj = type_string(obj)
if t_str:
return {t_str: obj}
elif isinstance(obj, Sequence):
return reduce(assign, [typify(item) for item in obj], {})
elif isinstance(obj, dict):
return {k: typify(v) for k, v in obj.items()}
if __name__ == "__main__":
Args.logdir = expanduser("~/runs")
cwd = expanduser("~/runs")
# _ = find_experiments(cwd=)
parameter_files = find_files(cwd, "*/**/parameters.pkl", stop=500 if DEBUG else None, show_progress=True)
print(len(parameter_files))
from elasticsearch import Elasticsearch
es = Elasticsearch([dict(host='localhost', port=9200)])
response = es.indices.delete(index='ml-dash', ignore=[400, 404])
cprint('deleted the index', 'green')
# note: how th nested index works
# https://www.elastic.co/guide/en/elasticsearch/reference/current/nested.html
response = es.indices.create(
index="ml-dash",
body=dict(mappings=dict(
dynamic=False,