def subcmd_commit(command_data):
client_name = command_data["subcmd_args"][0] if len(
command_data["subcmd_args"]) > 0 else None
if client_name == None:
print "Need client name to delete algs for"
return
zkroot = command_data["zkdetails"]["zkroot"]
if not is_existing_client(zkroot, client_name):
print "Invalid client[{client_name}]".format(**locals())
return
zk_client = command_data["zkdetails"]["zk_client"]
zkroot = command_data["zkdetails"]["zkroot"]
data_fpath = zkroot + gdata[
"all_clients_node_path"] + "/" + client_name + "/algs/_data_"
if not os.path.isfile(data_fpath):
"Data to commit not found!!"
f = open(data_fpath)
data_json = f.read()
f.close()
zk_client = command_data["zkdetails"]["zk_client"]
node_path = gdata["all_clients_node_path"] + "/" + client_name + "/algs"
zk_utils.node_set(zk_client, node_path, data_json)
def action_configure(command_data, opts):
client_name = opts.client_name
if (client_name == None) or (len(client_name) == 0):
print "Need client name to comfigure explanations for"
sys.exit(1)
zkroot = command_data["zkdetails"]["zkroot"]
if not is_existing_client(zkroot, client_name):
print "Invalid client[{client_name}]".format(**locals())
sys.exit(1)
zk_client = command_data["zkdetails"]["zk_client"]
ensure_client_has_recommendation_explanation(zkroot, zk_client,
client_name)
data_fpath = zkroot + gdata[
"all_clients_node_path"] + "/" + client_name + "/recommendation_explanation/_data_"
f = open(data_fpath)
json = f.read()
f.close()
data = seldon_utils.json_to_dict(json)
if opts.cache_enabled != None:
data["cache_enabled"] = True if opts.cache_enabled == "true" else False
if opts.default_locale != None:
data["default_locale"] = opts.default_locale
if opts.explanations_enabled != None:
data[
"explanations_enabled"] = True if opts.explanations_enabled == "true" else False
write_data_to_file(data_fpath, data)
node_path = gdata[
"all_clients_node_path"] + "/" + client_name + "/recommendation_explanation"
zk_utils.node_set(zk_client, node_path, seldon_utils.dict_to_json(data))
action_show(command_data, opts)
def action_edit(command_data, opts):
client_name = opts.client_name
if client_name == None:
print "Need client name to show the attr for"
sys.exit(1)
zkroot = command_data["zkdetails"]["zkroot"]
if not is_existing_client(zkroot, client_name):
print "Invalid client[{client_name}]".format(**locals())
sys.exit(1)
zk_client = command_data["zkdetails"]["zk_client"]
ensure_client_has_attr(zkroot, zk_client, client_name)
data_fpath = zkroot + gdata[
"all_clients_node_path"] + "/" + client_name + "/attr/_data_"
#do the edit
from subprocess import call
editor = seldon_utils.get_editor()
call([editor, data_fpath])
f = open(data_fpath)
json = f.read()
f.close()
data = seldon_utils.json_to_dict(json)
if data is None:
print "Invalid attr json!"
else:
write_data_to_file(data_fpath, data)
node_path = gdata["all_clients_node_path"] + "/" + client_name + "/attr"
zk_utils.node_set(zk_client, node_path,
seldon_utils.dict_to_json(data))
show_attr(data)
def add_client(zk_client, zkroot, client_name, db_name, consumer_details=None):
data_fpath = zkroot + "/config/dbcp/_data_"
f = open(data_fpath)
json = f.read()
data = json_to_dict(json)
f.close()
db_info = None
for db_info_entry in data['dbs']:
if db_info_entry['name'] == db_name:
db_info = db_info_entry
break
if db_info == None:
print "Invalid db name[{db_name}]".format(**locals())
return
dbSettings = {}
dbSettings["host"]=re.search('://(.*?):(.*?),',db_info["jdbc"]).groups()[0]
dbSettings["user"]=db_info["user"]
dbSettings["password"]=db_info["password"]
seldon_utils.addApiDb(db_name, dbSettings)
seldon_utils.addClientDb(client_name, dbSettings, consumer_details=None)
# write to local file
data_fpath = get_data_fpath(zkroot, client_name)
data = {'DB_JNDI_NAME': db_name}
write_data_to_file(data_fpath, data)
# write to zookeeper
node_path=gdata["all_clients_node_path"]+"/"+client_name
data_json = dict_to_json(data)
zk_utils.node_set(zk_client, node_path, data_json)
def action_configure(command_data, opts):
client_name = opts.client_name
if (client_name == None) or (len(client_name) == 0):
print "Need client name to comfigure explanations for"
sys.exit(1)
zkroot = command_data["zkdetails"]["zkroot"]
if not is_existing_client(zkroot, client_name):
print "Invalid client[{client_name}]".format(**locals())
sys.exit(1)
zk_client = command_data["zkdetails"]["zk_client"]
ensure_client_has_recommendation_explanation(zkroot, zk_client, client_name)
data_fpath = zkroot + gdata["all_clients_node_path"] + "/" + client_name + "/recommendation_explanation/_data_"
f = open(data_fpath)
json = f.read()
f.close()
data = seldon_utils.json_to_dict(json)
if opts.cache_enabled != None:
data["cache_enabled"] = True if opts.cache_enabled == "true" else False
if opts.default_locale!= None:
data["default_locale"] = opts.default_locale
if opts.explanations_enabled != None:
data["explanations_enabled"] = True if opts.explanations_enabled == "true" else False
write_data_to_file(data_fpath, data)
node_path = gdata["all_clients_node_path"]+"/"+client_name+"/recommendation_explanation"
zk_utils.node_set(zk_client, node_path, seldon_utils.dict_to_json(data))
action_show(command_data, opts)
def action_commit(command_data, opts):
client_name = opts.client_name
if client_name == None:
print "Need client name to add algs for"
sys.exit(1)
zkroot = command_data["zkdetails"]["zkroot"]
if not is_existing_client(zkroot, client_name):
print "Invalid client[{client_name}]".format(**locals())
return
zk_client = command_data["zkdetails"]["zk_client"]
zkroot = command_data["zkdetails"]["zkroot"]
data_fpath = zkroot + gdata[
"all_clients_node_path"] + "/" + client_name + "/predict_algs/_data_"
if not os.path.isfile(data_fpath):
"Data to commit not found!!"
f = open(data_fpath)
data_json = f.read()
f.close()
zk_client = command_data["zkdetails"]["zk_client"]
node_path = gdata[
"all_clients_node_path"] + "/" + client_name + "/predict_algs"
zk_utils.node_set(zk_client, node_path, data_json)
def action_edit(command_data, opts):
client_name = opts.client_name
if client_name == None:
print "Need client name to show the attr for"
sys.exit(1)
zkroot = command_data["zkdetails"]["zkroot"]
if not is_existing_client(zkroot, client_name):
print "Invalid client[{client_name}]".format(**locals())
sys.exit(1)
zk_client = command_data["zkdetails"]["zk_client"]
ensure_client_has_attr(zkroot, zk_client, client_name)
data_fpath = zkroot + gdata["all_clients_node_path"] + "/" + client_name + "/attr/_data_"
#do the edit
from subprocess import call
editor=seldon_utils.get_editor()
call([editor, data_fpath])
f = open(data_fpath)
json = f.read()
f.close()
data = seldon_utils.json_to_dict(json)
if data is None:
print "Invalid attr json!"
else:
write_data_to_file(data_fpath, data)
node_path = gdata["all_clients_node_path"]+"/"+client_name+"/attr"
zk_utils.node_set(zk_client, node_path, seldon_utils.dict_to_json(data))
show_attr(data)
def subcmd_add(command_data):
client_name = command_data["subcmd_args"][0] if len(
command_data["subcmd_args"]) > 0 else None
if client_name == None:
print "Need client name to add model for"
return
zkroot = command_data["zkdetails"]["zkroot"]
if not is_existing_client(zkroot, client_name):
print "Invalid client[{client_name}]".format(**locals())
return
default_models = command_data["default_models"]
models = default_models.keys()
for idx, model in enumerate(models):
print " [{idx}] {model}".format(**locals())
q = "Choose model: "
model_idx = None
try:
model_idx = int(raw_input(q))
except ValueError:
pass
model_name = models[
model_idx] if model_idx != None and model_idx >= 0 and model_idx < len(
models) else None
if model_name == None:
print "Invaild model"
return
data_fpath = "{zkroot}{all_clients_node_path}/{client_name}/offline/{model_name}/_data_".format(
zkroot=zkroot,
all_clients_node_path=gdata["all_clients_node_path"],
client_name=client_name,
model_name=model_name)
zk_client = command_data["zkdetails"]["zk_client"]
if not os.path.isfile(data_fpath):
node_path = "{all_clients_node_path}/{client_name}/offline/{model_name}".format(
all_clients_node_path=gdata["all_clients_node_path"],
client_name=client_name,
model_name=model_name)
if zk_client.exists(node_path):
write_node_value_to_file(zk_client, zkroot, node_path)
else:
default_model_data = default_models[model_name]["config"]
if default_model_data.has_key("inputPath"):
default_model_data["inputPath"] = command_data["seldon_models"]
if default_model_data.has_key("outputPath"):
default_model_data["outputPath"] = command_data[
"seldon_models"]
data = default_model_data
write_data_to_file(data_fpath, data)
zk_utils.node_set(zk_client, node_path,
seldon_utils.dict_to_json(data))
else:
print "Model [{model_name}] already added".format(**locals())
def subcmd_edit(command_data):
def have_client_name():
if client_name == None:
print "Need client name to edit model for"
return False
else:
return True
def have_valid_client_name():
if not is_existing_client(zkroot, client_name):
print "Invalid client[{client_name}]".format(**locals())
return False
else:
return True
client_name = command_data["subcmd_args"][0] if len(command_data["subcmd_args"])>0 else None
if not have_client_name(): return
zk_client = command_data["zkdetails"]["zk_client"]
zkroot = command_data["zkdetails"]["zkroot"]
if not have_valid_client_name(): return
models_for_client_fpath = "{zkroot}{all_clients_node_path}/{client_name}/offline".format(zkroot=zkroot,all_clients_node_path=gdata["all_clients_node_path"],client_name=client_name)
models = os.listdir(models_for_client_fpath)
for idx,model in enumerate(models):
print " [{idx}] {model}".format(**locals())
q="Choose model: "
model_idx = None
try:
model_idx = int(raw_input(q))
except ValueError:
pass
model_name = models[model_idx] if model_idx != None and model_idx>=0 and model_idx<len(models) else None
if model_name == None:
print "Invaild model"
return
data_fpath = "{zkroot}{all_clients_node_path}/{client_name}/offline/{model_name}/_data_".format(zkroot=zkroot,all_clients_node_path=gdata["all_clients_node_path"],client_name=client_name,model_name=model_name)
#do the edit
editor=seldon_utils.get_editor()
call([editor, data_fpath])
f = open(data_fpath)
json = f.read()
f.close()
data = seldon_utils.json_to_dict(json)
if data is None:
print "Invalid model json!"
else:
write_data_to_file(data_fpath, data)
node_path = "{all_clients_node_path}/{client_name}/offline/{model_name}".format(all_clients_node_path=gdata["all_clients_node_path"],client_name=client_name,model_name=model_name)
pp(node_path)
zk_utils.node_set(zk_client, node_path, seldon_utils.dict_to_json(data))
def action_edit(command_data, opts, extra_args):
zkroot = command_data["zkdetails"]["zkroot"]
def get_valid_client():
client_name = opts.client_name
if client_name == None:
print "Need client name to show models for"
sys.exit(1)
if not is_existing_client(zkroot, client_name):
print "Invalid client[{client_name}]".format(**locals())
sys.exit(1)
return client_name
def get_valid_model():
model_name = opts.model_name
if model_name == None:
print "Need model name to use"
sys.exit(1)
default_models = command_data["conf_data"]["default_models"]
if model_name not in default_models.keys():
print "Invalid model name: {model_name}".format(**locals())
sys.exit(1)
return model_name
client_name = get_valid_client()
model_name = get_valid_model()
zk_client = command_data["zkdetails"]["zk_client"]
data_fpath = "{zkroot}{all_clients_node_path}/{client_name}/offline/{model_name}/_data_".format(
zkroot=zkroot,
all_clients_node_path=gdata["all_clients_node_path"],
client_name=client_name,
model_name=model_name)
#do the edit
editor = seldon_utils.get_editor()
call([editor, data_fpath])
f = open(data_fpath)
json = f.read()
f.close()
data = seldon_utils.json_to_dict(json)
if data is None:
print "Invalid model json!"
else:
write_data_to_file(data_fpath, data)
node_path = "{all_clients_node_path}/{client_name}/offline/{model_name}".format(
all_clients_node_path=gdata["all_clients_node_path"],
client_name=client_name,
model_name=model_name)
pp(node_path)
zk_utils.node_set(zk_client, node_path,
seldon_utils.dict_to_json(data))
def ensure_client_has_algs(zk_client):
node_path = gdata["all_clients_node_path"] + "/" + client_name + "/algs"
if zk_client.exists(node_path):
write_node_value_to_file(zk_client, zkroot, node_path)
else:
default_alg_json = '{"algorithms":[{"config":[],"filters":[],"includers":[],"name":"recentItemsRecommender"}],"combiner":"firstSuccessfulCombiner"}'
data = json_to_dict(default_alg_json)
write_data_to_file(data_fpath, data)
zk_utils.node_set(zk_client, node_path, dict_to_json(data))
def ensure_client_has_algs(zkroot, zk_client, client_name):
data_fpath = "{zkroot}{all_clients_node_path}/{client_name}/predict_algs/_data_".format(zkroot=zkroot,all_clients_node_path=gdata["all_clients_node_path"],client_name=client_name)
if not os.path.isfile(data_fpath):
node_path = gdata["all_clients_node_path"]+"/"+client_name+"/predict_algs"
if zk_client.exists(node_path):
write_node_value_to_file(zk_client, zkroot, node_path)
else:
default_alg_json = '{"algorithms":[]}'
data = json_to_dict(default_alg_json)
write_data_to_file(data_fpath, data)
zk_utils.node_set(zk_client, node_path, dict_to_json(data))
def action_set(command_data, opts):
if "grpc_util_home" in command_data["conf_data"]:
grpc_home = command_data["conf_data"]["grpc_util_home"]
else:
print "Need the location for grpc_util_home to create proto jar via maven"
sys.exit(1)
client_name = opts.client_name
if client_name == None:
print "Need client name to show the rpc settings for"
sys.exit(1)
proto_file = opts.proto
if proto_file == None:
print "Need --proto argument"
sys.exit(1)
request_class = opts.request_class
if request_class == None:
print "Need request class to setup!"
sys.exit(1)
jar_file = "/seldon-data/rpc/jar/" + client_name + ".jar"
if not create_jar(grpc_home, opts.proto, jar_file) == 0:
print "Failed to create jar file from proto"
sys.exit(1)
zkroot = command_data["zkdetails"]["zkroot"]
zk_client = command_data["zkdetails"]["zk_client"]
if opts.reply_class == None:
data = {
"jarFilename": jar_file,
"requestClassName": opts.request_class
}
else:
data = {
"jarFilename": jar_file,
"requestClassName": opts.request_class,
"replyClassName": opts.reply_class
}
data_fpath = zkroot + gdata[
"all_clients_node_path"] + "/" + client_name + "/rpc/_data_"
node_path = gdata["all_clients_node_path"] + "/" + client_name + "/rpc"
write_data_to_file(data_fpath, data)
f = open(data_fpath)
data_json = f.read()
f.close()
zk_client = command_data["zkdetails"]["zk_client"]
zk_utils.node_set(zk_client, node_path, data_json)
def ensure_client_has_recommendation_explanation(zkroot, zk_client, client_name):
data_fpath = "{zkroot}{all_clients_node_path}/{client_name}/recommendation_explanation/_data_".format(zkroot=zkroot,all_clients_node_path=gdata["all_clients_node_path"],client_name=client_name)
if not os.path.isfile(data_fpath):
node_path = gdata["all_clients_node_path"]+"/"+client_name+"/recommendation_explanation"
if zk_client.exists(node_path):
write_node_value_to_file(zk_client, zkroot, node_path)
else:
default_rec_exp = '{"cache_enabled":true,"default_locale":"us-en","explanations_enabled":false}'
data = seldon_utils.json_to_dict(default_rec_exp)
write_data_to_file(data_fpath, data)
zk_utils.node_set(zk_client, node_path, seldon_utils.dict_to_json(data))
def ensure_client_has_algs(zkroot, zk_client, client_name):
data_fpath = "{zkroot}{all_clients_node_path}/{client_name}/algs/_data_".format(zkroot=zkroot,all_clients_node_path=gdata["all_clients_node_path"],client_name=client_name)
if not os.path.isfile(data_fpath):
node_path = gdata["all_clients_node_path"]+"/"+client_name+"/algs"
if zk_client.exists(node_path):
write_node_value_to_file(zk_client, zkroot, node_path)
else:
default_alg_json = '{"algorithms":[{"config":[],"filters":[],"includers":[],"name":"recentItemsRecommender"}],"combiner":"firstSuccessfulCombiner"}'
data = json_to_dict(default_alg_json)
write_data_to_file(data_fpath, data)
zk_utils.node_set(zk_client, node_path, dict_to_json(data))
def ensure_client_has_algs(zkroot, zk_client, client_name):
data_fpath = "{zkroot}{all_clients_node_path}/{client_name}/predict_algs/_data_".format(zkroot=zkroot,all_clients_node_path=gdata["all_clients_node_path"],client_name=client_name)
if not os.path.isfile(data_fpath):
node_path = gdata["all_clients_node_path"]+"/"+client_name+"/predict_algs"
if zk_client.exists(node_path):
write_node_value_to_file(zk_client, zkroot, node_path)
else:
default_alg_json = '{"algorithms":[]}'
data = json_to_dict(default_alg_json)
write_data_to_file(data_fpath, data)
zk_utils.node_set(zk_client, node_path, dict_to_json(data))
def action_add(command_data, opts, extra_args):
client_name = opts.client_name
if client_name == None:
print "Need client name to add model for"
sys.exit(1)
zkroot = command_data["zkdetails"]["zkroot"]
if not is_existing_client(zkroot, client_name):
print "Invalid client[{client_name}]".format(**locals())
sys.exit(1)
model_name = opts.model_name
if model_name == None:
print "Need model name to use"
sys.exit(1)
default_models = command_data["conf_data"]["default_models"]
if model_name not in default_models.keys():
print "Invalid model name: {model_name}".format(**locals())
sys.exit(1)
config_args = get_config_args(extra_args)
data_fpath = "{zkroot}{all_clients_node_path}/{client_name}/offline/{model_name}/_data_".format(zkroot=zkroot,all_clients_node_path=gdata["all_clients_node_path"],client_name=client_name,model_name=model_name)
zk_client = command_data["zkdetails"]["zk_client"]
node_path = "{all_clients_node_path}/{client_name}/offline/{model_name}".format(all_clients_node_path=gdata["all_clients_node_path"],client_name=client_name,model_name=model_name)
if not os.path.isfile(data_fpath):
if zk_client.exists(node_path):
write_node_value_to_file(zk_client, zkroot, node_path)
f = open(data_fpath)
json = f.read()
f.close()
data = seldon_utils.json_to_dict(json)
else:
default_model_data = default_models[model_name]["config"]
if default_model_data.has_key("inputPath"):
default_model_data["inputPath"]=command_data["conf_data"]["seldon_models"]
if default_model_data.has_key("outputPath"):
default_model_data["outputPath"]=command_data["conf_data"]["seldon_models"]
data = default_model_data
else:
f = open(data_fpath)
json = f.read()
f.close()
data = seldon_utils.json_to_dict(json)
print "Model [{model_name}] already added".format(**locals())
for k in config_args:
print "adding config ",k,":",config_args[k]
data[k] = config_args[k]
write_data_to_file(data_fpath, data)
zk_utils.node_set(zk_client, node_path, seldon_utils.dict_to_json(data))
def action_commit(command_data, opts):
zkroot = command_data["zkdetails"]["zkroot"]
data_fpath = zkroot + gdata['data_path']
if not os.path.isfile(data_fpath):
write_data_to_file(data_fpath, gdata["default_data"])
f = open(data_fpath)
data_json = f.read()
f.close()
zk_client=command_data["zkdetails"]["zk_client"]
node_path=gdata["node_path"]
zk_utils.node_set(zk_client, node_path, data_json)
def ensure_client_has_attr(zkroot, zk_client, client_name):
data_fpath = "{zkroot}{all_clients_node_path}/{client_name}/attr/_data_".format(
zkroot=zkroot,all_clients_node_path=gdata["all_clients_node_path"],client_name=client_name)
if not os.path.isfile(data_fpath):
node_path = gdata["all_clients_node_path"]+"/"+client_name+"/attr"
if zk_client.exists(node_path):
write_node_value_to_file(zk_client, zkroot, node_path)
else:
default_attr_json = '{"types":[{"type_attrs":[{"name":"title","value_type":"string"}],"type_id":1,"type_name":"defaulttype"}]}'
data = seldon_utils.json_to_dict(default_attr_json)
write_data_to_file(data_fpath, data)
zk_utils.node_set(zk_client, node_path, seldon_utils.dict_to_json(data))
def action_commit(command_data, opts):
zkroot = command_data["zkdetails"]["zkroot"]
data_fpath = zkroot + gdata['data_path']
if not os.path.isfile(data_fpath):
write_data_to_file(data_fpath, gdata["default_data"])
f = open(data_fpath)
data_json = f.read()
f.close()
zk_client = command_data["zkdetails"]["zk_client"]
node_path = gdata["node_path"]
zk_utils.node_set(zk_client, node_path, data_json)
def subcmd_commit(command_data):
zkroot = command_data["zkdetails"]["zkroot"]
zk_client = command_data["zkdetails"]["zk_client"]
data_fpath = zkroot + gdata['data_path']
ensure_local_data_file_exists(zk_client, zkroot)
f = open(data_fpath)
data_json = f.read()
f.close()
zk_client=command_data["zkdetails"]["zk_client"]
node_path=gdata["node_path"]
zk_utils.node_set(zk_client, node_path, data_json)
def subcmd_commit(command_data):
zkroot = command_data["zkdetails"]["zkroot"]
zk_client = command_data["zkdetails"]["zk_client"]
data_fpath = zkroot + gdata['data_path']
ensure_local_data_file_exists(zk_client, zkroot)
f = open(data_fpath)
data_json = f.read()
f.close()
zk_client = command_data["zkdetails"]["zk_client"]
node_path = gdata["node_path"]
zk_utils.node_set(zk_client, node_path, data_json)
def action_edit(command_data, opts):
zkroot = command_data["zkdetails"]["zkroot"]
def get_valid_client():
client_name = opts.client_name
if client_name == None:
print "Need client name to show models for"
sys.exit(1)
if not is_existing_client(zkroot, client_name):
print "Invalid client[{client_name}]".format(**locals())
sys.exit(1)
return client_name
def get_valid_model():
model_name = opts.model_name
if model_name == None:
print "Need model name to use"
sys.exit(1)
default_models = command_data["conf_data"]["default_models"]
if model_name not in default_models.keys():
print "Invalid model name: {model_name}".format(**locals())
sys.exit(1)
return model_name
client_name = get_valid_client()
model_name = get_valid_model()
zk_client = command_data["zkdetails"]["zk_client"]
data_fpath = "{zkroot}{all_clients_node_path}/{client_name}/offline/{model_name}/_data_".format(zkroot=zkroot,all_clients_node_path=gdata["all_clients_node_path"],client_name=client_name,model_name=model_name)
#do the edit
editor=seldon_utils.get_editor()
call([editor, data_fpath])
f = open(data_fpath)
json = f.read()
f.close()
data = seldon_utils.json_to_dict(json)
if data is None:
print "Invalid model json!"
else:
write_data_to_file(data_fpath, data)
node_path = "{all_clients_node_path}/{client_name}/offline/{model_name}".format(all_clients_node_path=gdata["all_clients_node_path"],client_name=client_name,model_name=model_name)
pp(node_path)
zk_utils.node_set(zk_client, node_path, seldon_utils.dict_to_json(data))
def action_set(command_data, opts):
if "grpc_util_home" in command_data["conf_data"]:
grpc_home = command_data["conf_data"]["grpc_util_home"]
else:
print "Need the location for grpc_util_home to create proto jar via maven"
sys.exit(1)
client_name = opts.client_name
if client_name == None:
print "Need client name to show the rpc settings for"
sys.exit(1)
proto_file = opts.proto
if proto_file == None:
print "Need --proto argument"
sys.exit(1)
request_class = opts.request_class
if request_class == None:
print "Need request class to setup!"
sys.exit(1)
jar_file = "/seldon-data/rpc/jar/"+client_name+".jar"
if not create_jar(grpc_home,opts.proto,jar_file) == 0:
print "Failed to create jar file from proto"
sys.exit(1)
zkroot = command_data["zkdetails"]["zkroot"]
zk_client = command_data["zkdetails"]["zk_client"]
if opts.reply_class == None:
data = {"jarFilename":jar_file,"requestClassName":opts.request_class}
else:
data = {"jarFilename":jar_file,"requestClassName":opts.request_class,"replyClassName":opts.reply_class}
data_fpath = zkroot + gdata["all_clients_node_path"] + "/" + client_name + "/rpc/_data_"
node_path = gdata["all_clients_node_path"] + "/" + client_name + "/rpc"
write_data_to_file(data_fpath, data)
f = open(data_fpath)
data_json = f.read()
f.close()
zk_client=command_data["zkdetails"]["zk_client"]
zk_utils.node_set(zk_client, node_path, data_json)
def ensure_client_has_recommendation_explanation(zkroot, zk_client,
client_name):
data_fpath = "{zkroot}{all_clients_node_path}/{client_name}/recommendation_explanation/_data_".format(
zkroot=zkroot,
all_clients_node_path=gdata["all_clients_node_path"],
client_name=client_name)
if not os.path.isfile(data_fpath):
node_path = gdata[
"all_clients_node_path"] + "/" + client_name + "/recommendation_explanation"
if zk_client.exists(node_path):
write_node_value_to_file(zk_client, zkroot, node_path)
else:
default_rec_exp = '{"cache_enabled":true,"default_locale":"us-en","explanations_enabled":false}'
data = seldon_utils.json_to_dict(default_rec_exp)
write_data_to_file(data_fpath, data)
zk_utils.node_set(zk_client, node_path,
seldon_utils.dict_to_json(data))
def store_json(command_data,opts):
f = open(opts.json)
json = f.read()
f.close()
data = seldon_utils.json_to_dict(json)
zk_client = command_data["zkdetails"]["zk_client"]
zkroot = command_data["zkdetails"]["zkroot"]
data_fpath = zkroot + gdata["all_clients_node_path"] + "/" + opts.client_name + "/attr/_data_"
if data is None:
print "Invalid attr json!"
sys.exit(1)
else:
write_data_to_file(data_fpath, data)
node_path = gdata["all_clients_node_path"]+"/"+opts.client_name+"/attr"
zk_utils.node_set(zk_client, node_path, seldon_utils.dict_to_json(data))
def subcmd_add(command_data):
client_name = command_data["subcmd_args"][0] if len(command_data["subcmd_args"])>0 else None
if client_name == None:
print "Need client name to add model for"
return
zkroot = command_data["zkdetails"]["zkroot"]
if not is_existing_client(zkroot, client_name):
print "Invalid client[{client_name}]".format(**locals())
return
default_models = command_data["default_models"]
models = default_models.keys()
for idx,model in enumerate(models):
print " [{idx}] {model}".format(**locals())
q="Choose model: "
model_idx = None
try:
model_idx = int(raw_input(q))
except ValueError:
pass
model_name = models[model_idx] if model_idx != None and model_idx>=0 and model_idx<len(models) else None
if model_name == None:
print "Invaild model"
return
data_fpath = "{zkroot}{all_clients_node_path}/{client_name}/offline/{model_name}/_data_".format(zkroot=zkroot,all_clients_node_path=gdata["all_clients_node_path"],client_name=client_name,model_name=model_name)
zk_client = command_data["zkdetails"]["zk_client"]
if not os.path.isfile(data_fpath):
node_path = "{all_clients_node_path}/{client_name}/offline/{model_name}".format(all_clients_node_path=gdata["all_clients_node_path"],client_name=client_name,model_name=model_name)
if zk_client.exists(node_path):
write_node_value_to_file(zk_client, zkroot, node_path)
else:
default_model_data = default_models[model_name]["config"]
if default_model_data.has_key("inputPath"):
default_model_data["inputPath"]=command_data["seldon_models"]
if default_model_data.has_key("outputPath"):
default_model_data["outputPath"]=command_data["seldon_models"]
data = default_model_data
write_data_to_file(data_fpath, data)
zk_utils.node_set(zk_client, node_path, seldon_utils.dict_to_json(data))
else:
print "Model [{model_name}] already added".format(**locals())
def action_set(command_data, opts):
client_name = opts.client_name
if client_name == None:
print "Need client name to show the rpc settings for"
sys.exit(1)
jar_file = opts.jar
if jar_file == None:
print "Need protobuf jar file location to setup!"
sys.exit(1)
request_class = opts.request_class
if request_class == None:
print "Need request class to setup!"
sys.exit(1)
zkroot = command_data["zkdetails"]["zkroot"]
zk_client = command_data["zkdetails"]["zk_client"]
if opts.reply_class == None:
data = {
"jarFilename": opts.jar,
"requestClassName": opts.request_class
}
else:
data = {
"jarFilename": opts.jar,
"requestClassName": opts.request_class,
"replyClassName": opts.reply_class
}
data_fpath = zkroot + gdata[
"all_clients_node_path"] + "/" + client_name + "/rpc/_data_"
node_path = gdata["all_clients_node_path"] + "/" + client_name + "/rpc"
write_data_to_file(data_fpath, data)
f = open(data_fpath)
data_json = f.read()
f.close()
zk_client = command_data["zkdetails"]["zk_client"]
zk_utils.node_set(zk_client, node_path, data_json)
def action_commit(command_data, opts):
client_name = opts.client_name
if client_name == None:
print "Need client name to add algs for"
sys.exit(1)
zkroot = command_data["zkdetails"]["zkroot"]
if not is_existing_client(zkroot, client_name):
print "Invalid client[{client_name}]".format(**locals())
return
zk_client = command_data["zkdetails"]["zk_client"]
zkroot = command_data["zkdetails"]["zkroot"]
data_fpath = zkroot + gdata[
"all_clients_node_path"] + "/" + client_name + "/algs/_data_"
if not os.path.isfile(data_fpath):
"Data to commit not found!!"
f = open(data_fpath)
data_json = f.read()
f.close()
zk_client = command_data["zkdetails"]["zk_client"]
node_path = gdata["all_clients_node_path"] + "/" + client_name + "/algs"
zk_utils.node_set(zk_client, node_path, data_json)
# activate any required models
default_algorithms = command_data["conf_data"]["default_algorithms"]
data = json_to_dict(data_json)
algorithms = data["algorithms"]
print "algorithms:"
for alg in algorithms:
alg_name = alg["name"]
if default_algorithms[alg_name].has_key("zk_activate_node"):
node_path = default_algorithms[alg_name]["zk_activate_node"]
node_fpath = zkroot + node_path + "/_data_"
data_models = get_data_from_file(node_fpath)
zk_utils.node_set(zk_client, node_path, data_models)
def action_commit(command_data, opts):
client_name = opts.client_name
if client_name == None:
print "Need client name to add algs for"
sys.exit(1)
zkroot = command_data["zkdetails"]["zkroot"]
if not is_existing_client(zkroot, client_name):
print "Invalid client[{client_name}]".format(**locals())
return
zk_client = command_data["zkdetails"]["zk_client"]
zkroot = command_data["zkdetails"]["zkroot"]
data_fpath = zkroot + gdata["all_clients_node_path"] + "/" + client_name + "/predict_algs/_data_"
if not os.path.isfile(data_fpath):
"Data to commit not found!!"
f = open(data_fpath)
data_json = f.read()
f.close()
zk_client = command_data["zkdetails"]["zk_client"]
node_path = gdata["all_clients_node_path"] + "/" + client_name + "/predict_algs"
zk_utils.node_set(zk_client, node_path, data_json)
def subcmd_commit(command_data):
client_name = command_data["subcmd_args"][0] if len(command_data["subcmd_args"])>0 else None
if client_name == None:
print "Need client name to delete algs for"
return
zkroot = command_data["zkdetails"]["zkroot"]
if not is_existing_client(zkroot, client_name):
print "Invalid client[{client_name}]".format(**locals())
return
zk_client = command_data["zkdetails"]["zk_client"]
zkroot = command_data["zkdetails"]["zkroot"]
data_fpath = zkroot + gdata["all_clients_node_path"] + "/" + client_name + "/algs/_data_"
if not os.path.isfile(data_fpath):
"Data to commit not found!!"
f = open(data_fpath)
data_json = f.read()
f.close()
zk_client = command_data["zkdetails"]["zk_client"]
node_path = gdata["all_clients_node_path"] + "/" + client_name + "/algs"
zk_utils.node_set(zk_client, node_path, data_json)
def action_commit(command_data, opts):
client_name = opts.client_name
if client_name == None:
print "Need client name to add algs for"
sys.exit(1)
zkroot = command_data["zkdetails"]["zkroot"]
if not is_existing_client(zkroot, client_name):
print "Invalid client[{client_name}]".format(**locals())
return
zk_client = command_data["zkdetails"]["zk_client"]
zkroot = command_data["zkdetails"]["zkroot"]
data_fpath = zkroot + gdata["all_clients_node_path"] + "/" + client_name + "/algs/_data_"
if not os.path.isfile(data_fpath):
"Data to commit not found!!"
f = open(data_fpath)
data_json = f.read()
f.close()
zk_client = command_data["zkdetails"]["zk_client"]
node_path = gdata["all_clients_node_path"] + "/" + client_name + "/algs"
zk_utils.node_set(zk_client, node_path, data_json)
# activate any required models
default_algorithms = command_data["conf_data"]["default_algorithms"]
data = json_to_dict(data_json)
algorithms = data["algorithms"]
print "algorithms:"
for alg in algorithms:
alg_name=alg["name"]
if default_algorithms[alg_name].has_key("zk_activate_node"):
node_path = default_algorithms[alg_name]["zk_activate_node"]
node_fpath = zkroot + node_path + "/_data_"
data_models = get_data_from_file(node_fpath)
zk_utils.node_set(zk_client, node_path, data_models)
def action_commit(command_data, opts):
client_name = opts.client_name
if client_name == None:
print "Need client name to commit data for"
sys.exit(1)
zkroot = command_data["zkdetails"]["zkroot"]
if not is_existing_client(zkroot, client_name):
print "Invalid client[{client_name}]".format(**locals())
return
zk_client = command_data["zkdetails"]["zk_client"]
zkroot = command_data["zkdetails"]["zkroot"]
data_fpath = zkroot + gdata[
"all_clients_node_path"] + "/" + client_name + "/alg_rectags/_data_"
if os.path.isfile(data_fpath):
f = open(data_fpath)
data_json = f.read()
f.close()
zk_client = command_data["zkdetails"]["zk_client"]
node_path = gdata[
"all_clients_node_path"] + "/" + client_name + "/alg_rectags"
zk_utils.node_set(zk_client, node_path, data_json)
# activate any required models
data = json_to_dict(data_json)
recommender_set = set()
if data["defaultStrategy"].has_key("algorithms"):
for alg in data["defaultStrategy"]["algorithms"]:
recommender_set.add(alg["name"])
elif data["defaultStrategy"].has_key("variations"):
for variation in data["defaultStrategy"]["variations"]:
for alg in variation["config"]["algorithms"]:
recommender_set.add(alg["name"])
default_algorithms = command_data["conf_data"]["default_algorithms"]
for recommender_name in recommender_set:
if default_algorithms[recommender_name].has_key(
"zk_activate_node"):
print "activate", recommender_name
node_path = default_algorithms[recommender_name][
"zk_activate_node"]
node_fpath = zkroot + node_path + "/_data_"
data_models = get_data_from_file(node_fpath)
zk_utils.node_set(zk_client, node_path, data_models)
return
#TODO remove the following once only using alg_rectags
data_fpath = zkroot + gdata[
"all_clients_node_path"] + "/" + client_name + "/algs/_data_"
if not os.path.isfile(data_fpath):
print "Data to commit not found!!"
sys.exit(1)
f = open(data_fpath)
data_json = f.read()
f.close()
zk_client = command_data["zkdetails"]["zk_client"]
node_path = gdata["all_clients_node_path"] + "/" + client_name + "/algs"
zk_utils.node_set(zk_client, node_path, data_json)
# activate any required models
default_algorithms = command_data["conf_data"]["default_algorithms"]
data = json_to_dict(data_json)
algorithms = data["algorithms"]
print "algorithms:"
for alg in algorithms:
alg_name = alg["name"]
if default_algorithms[alg_name].has_key("zk_activate_node"):
node_path = default_algorithms[alg_name]["zk_activate_node"]
node_fpath = zkroot + node_path + "/_data_"
data_models = get_data_from_file(node_fpath)
zk_utils.node_set(zk_client, node_path, data_models)
def action_add(command_data, opts, extra_args):
client_name = opts.client_name
if client_name == None:
print "Need client name to add model for"
sys.exit(1)
zkroot = command_data["zkdetails"]["zkroot"]
if not is_existing_client(zkroot, client_name):
print "Invalid client[{client_name}]".format(**locals())
sys.exit(1)
model_name = opts.model_name
if model_name == None:
print "Need model name to use"
sys.exit(1)
default_models = command_data["conf_data"]["default_models"]
if model_name not in default_models.keys():
print "Invalid model name: {model_name}".format(**locals())
sys.exit(1)
config_args = get_config_args(extra_args)
data_fpath = "{zkroot}{all_clients_node_path}/{client_name}/offline/{model_name}/_data_".format(
zkroot=zkroot,
all_clients_node_path=gdata["all_clients_node_path"],
client_name=client_name,
model_name=model_name)
zk_client = command_data["zkdetails"]["zk_client"]
node_path = "{all_clients_node_path}/{client_name}/offline/{model_name}".format(
all_clients_node_path=gdata["all_clients_node_path"],
client_name=client_name,
model_name=model_name)
if not os.path.isfile(data_fpath):
if zk_client.exists(node_path):
write_node_value_to_file(zk_client, zkroot, node_path)
f = open(data_fpath)
json = f.read()
f.close()
data = seldon_utils.json_to_dict(json)
else:
default_model_data = default_models[model_name]["config"]
if default_model_data.has_key("inputPath"):
default_model_data["inputPath"] = command_data["conf_data"][
"seldon_models"]
if default_model_data.has_key("outputPath"):
default_model_data["outputPath"] = command_data["conf_data"][
"seldon_models"]
data = default_model_data
else:
f = open(data_fpath)
json = f.read()
f.close()
data = seldon_utils.json_to_dict(json)
print "Model [{model_name}] already added".format(**locals())
for k in config_args:
print "adding config ", k, ":", config_args[k]
data[k] = config_args[k]
write_data_to_file(data_fpath, data)
zk_utils.node_set(zk_client, node_path, seldon_utils.dict_to_json(data))
