def callback(self,
exp_ids,
num_episodes,
scores,
hparams,
max_scores,
_time,
max_score,
max_score_ind,
config={}):
opts_syncer_path = config.get("syncer_conf")
if opts_syncer_path:
syncer_path = os.path.abspath(opts_syncer_path)
self.log.debug(
"ModelDB callback - Loading config from file {}".format(
syncer_path))
else:
syncer_path = os.path.abspath(
os.path.join(os.path.dirname(__file__),
'../../../scaffold/modeldb/syncer.json'))
self.log.debug(
"ModelDB callback - Loading default config from file {}".
format(syncer_path))
for _arg in list(zip(exp_ids, hparams, max_scores, _time)):
model = Model("RL", 'CartPole-{}'.format(_arg[0]),
"https://github.com/hypnosapos/cartpole-rl-remote")
self.syncer.sync_datasets(
dict(gym=Dataset("/path/to/gym",
{"episodes": num_episodes[0]})))
model_config = ModelConfig("RL", _arg[1])
self.syncer.sync_model("gym", model_config, model)
model_metrics = ModelMetrics({'score': _arg[2]})
self.syncer.sync_metrics("gym", model, model_metrics)
self.syncer.sync()
args = parser.parse_args()
req_j = args.request
req = json.loads(req_j)
owner = req["owner"]
study = req["study"]
train = req["train"]
modelpath = req["modelpath"]
hyp = req["hyperparameter"]
met = req["metrics"]
syncer_obj = Syncer.create_syncer(study,
owner,
"",
host=args.server,
port=args.port)
datasets = {
"train": Dataset("", {}),
"test": Dataset("", {}),
}
model = train
mdb_model = Model(study, model, modelpath)
model_config = ModelConfig("NN", hyp)
model_metrics = ModelMetrics(met)
syncer_obj.sync_datasets(datasets)
syncer_obj.sync_model("train", model_config, mdb_model)
syncer_obj.sync_metrics("test", mdb_model, model_metrics)
syncer_obj.sync()
print("I'm training some model")
datasets = {
"train": Dataset("/path/to/train", {
"num_cols": 15,
"dist": "random"
}),
"test": Dataset("/path/to/test", {
"num_cols": 15,
"dist": "gaussian"
})
}
# create the Model, ModelConfig, and ModelMetrics instances
model = "model_obj"
model_type = "NN"
mdb_model1 = Model(model_type, model, "/path/to/model1")
model_config1 = ModelConfig(model_type, {"l1": 10})
model_metrics1 = ModelMetrics({"accuracy": 0.8})
# sync the datasets to modeldb
syncer_obj.sync_datasets(datasets)
# sync the model with its model config and specify which dataset tag to use for it
syncer_obj.sync_model("train", model_config1, mdb_model1)
# sync the metrics to the model and also specify which dataset tag to use for it
syncer_obj.sync_metrics("test", mdb_model1, model_metrics1)
syncer_obj.sync()
print("I'm training some model")
datasets = {
"train": Dataset("/path/to/train", {
"num_cols": 15,
"dist": "random"
}),
"test": Dataset("/path/to/test", {
"num_cols": 15,
"dist": "gaussian"
})
}
model = "model_obj"
mdb_model1 = Model("NN", model, "/path/to/model1")
model_config1 = ModelConfig("NN", {"l1": 10})
model_metrics1 = ModelMetrics({"accuracy": 0.8})
mdb_model2 = Model("NN", model, "/path/to/model2")
model_config2 = ModelConfig("NN", {"l1": 20})
model_metrics2 = ModelMetrics({"accuracy": 0.9})
syncer_obj.sync_datasets(datasets)
syncer_obj.sync_model("train", model_config1, mdb_model1)
syncer_obj.sync_metrics("test", mdb_model1, model_metrics1)
syncer_obj.sync_model("train", model_config2, mdb_model2)
syncer_obj.sync_metrics("test", mdb_model2, model_metrics2)
syncer_obj.sync()