def callback(self,
exp_ids,
num_episodes,
scores,
hparams,
max_scores,
_time,
max_score,
max_score_ind,
config={}):
opts_syncer_path = config.get("syncer_conf")
if opts_syncer_path:
syncer_path = os.path.abspath(opts_syncer_path)
self.log.debug(
"ModelDB callback - Loading config from file {}".format(
syncer_path))
else:
syncer_path = os.path.abspath(
os.path.join(os.path.dirname(__file__),
'../../../scaffold/modeldb/syncer.json'))
self.log.debug(
"ModelDB callback - Loading default config from file {}".
format(syncer_path))
for _arg in list(zip(exp_ids, hparams, max_scores, _time)):
model = Model("RL", 'CartPole-{}'.format(_arg[0]),
"https://github.com/hypnosapos/cartpole-rl-remote")
self.syncer.sync_datasets(
dict(gym=Dataset("/path/to/gym",
{"episodes": num_episodes[0]})))
model_config = ModelConfig("RL", _arg[1])
self.syncer.sync_model("gym", model_config, model)
model_metrics = ModelMetrics({'score': _arg[2]})
self.syncer.sync_metrics("gym", model, model_metrics)
self.syncer.sync()
args = parser.parse_args()
req_j = args.request
req = json.loads(req_j)
owner = req["owner"]
study = req["study"]
train = req["train"]
modelpath = req["modelpath"]
hyp = req["hyperparameter"]
met = req["metrics"]
syncer_obj = Syncer.create_syncer(study,
owner,
"",
host=args.server,
port=args.port)
datasets = {
"train": Dataset("", {}),
"test": Dataset("", {}),
}
model = train
mdb_model = Model(study, model, modelpath)
model_config = ModelConfig("NN", hyp)
model_metrics = ModelMetrics(met)
syncer_obj.sync_datasets(datasets)
syncer_obj.sync_model("train", model_config, mdb_model)
syncer_obj.sync_metrics("test", mdb_model, model_metrics)
syncer_obj.sync()
# Example: Create a syncer explicitly
# syncer_obj = Syncer(
# NewOrExistingProject("gensim test", "test_user",
# "using modeldb light logging"),
# DefaultExperiment(),
# NewExperimentRun("", "sha_A1B2C3D4"))
# Example: Create a syncer from an existing experiment run
# experiment_run_id = int(sys.argv[len(sys.argv) - 1])
# syncer_obj = Syncer.create_syncer_for_experiment_run(experiment_run_id)
print("I'm training some model")
datasets = {
"train": Dataset("/path/to/train", {
"num_cols": 15,
"dist": "random"
}),
"test": Dataset("/path/to/test", {
"num_cols": 15,
"dist": "gaussian"
})
}
# create the Model, ModelConfig, and ModelMetrics instances
model = "model_obj"
model_type = "NN"
mdb_model1 = Model(model_type, model, "/path/to/model1")
model_config1 = ModelConfig(model_type, {"l1": 10})
model_metrics1 = ModelMetrics({"accuracy": 0.8})
# sync the datasets to modeldb