def subcmd_commit(command_data): client_name = command_data["subcmd_args"][0] if len( command_data["subcmd_args"]) > 0 else None if client_name == None: print "Need client name to delete algs for" return zkroot = command_data["zkdetails"]["zkroot"] if not is_existing_client(zkroot, client_name): print "Invalid client[{client_name}]".format(**locals()) return zk_client = command_data["zkdetails"]["zk_client"] zkroot = command_data["zkdetails"]["zkroot"] data_fpath = zkroot + gdata[ "all_clients_node_path"] + "/" + client_name + "/algs/_data_" if not os.path.isfile(data_fpath): "Data to commit not found!!" f = open(data_fpath) data_json = f.read() f.close() zk_client = command_data["zkdetails"]["zk_client"] node_path = gdata["all_clients_node_path"] + "/" + client_name + "/algs" zk_utils.node_set(zk_client, node_path, data_json)
def action_configure(command_data, opts): client_name = opts.client_name if (client_name == None) or (len(client_name) == 0): print "Need client name to comfigure explanations for" sys.exit(1) zkroot = command_data["zkdetails"]["zkroot"] if not is_existing_client(zkroot, client_name): print "Invalid client[{client_name}]".format(**locals()) sys.exit(1) zk_client = command_data["zkdetails"]["zk_client"] ensure_client_has_recommendation_explanation(zkroot, zk_client, client_name) data_fpath = zkroot + gdata[ "all_clients_node_path"] + "/" + client_name + "/recommendation_explanation/_data_" f = open(data_fpath) json = f.read() f.close() data = seldon_utils.json_to_dict(json) if opts.cache_enabled != None: data["cache_enabled"] = True if opts.cache_enabled == "true" else False if opts.default_locale != None: data["default_locale"] = opts.default_locale if opts.explanations_enabled != None: data[ "explanations_enabled"] = True if opts.explanations_enabled == "true" else False write_data_to_file(data_fpath, data) node_path = gdata[ "all_clients_node_path"] + "/" + client_name + "/recommendation_explanation" zk_utils.node_set(zk_client, node_path, seldon_utils.dict_to_json(data)) action_show(command_data, opts)
def action_edit(command_data, opts): client_name = opts.client_name if client_name == None: print "Need client name to show the attr for" sys.exit(1) zkroot = command_data["zkdetails"]["zkroot"] if not is_existing_client(zkroot, client_name): print "Invalid client[{client_name}]".format(**locals()) sys.exit(1) zk_client = command_data["zkdetails"]["zk_client"] ensure_client_has_attr(zkroot, zk_client, client_name) data_fpath = zkroot + gdata[ "all_clients_node_path"] + "/" + client_name + "/attr/_data_" #do the edit from subprocess import call editor = seldon_utils.get_editor() call([editor, data_fpath]) f = open(data_fpath) json = f.read() f.close() data = seldon_utils.json_to_dict(json) if data is None: print "Invalid attr json!" else: write_data_to_file(data_fpath, data) node_path = gdata["all_clients_node_path"] + "/" + client_name + "/attr" zk_utils.node_set(zk_client, node_path, seldon_utils.dict_to_json(data)) show_attr(data)
def add_client(zk_client, zkroot, client_name, db_name, consumer_details=None): data_fpath = zkroot + "/config/dbcp/_data_" f = open(data_fpath) json = f.read() data = json_to_dict(json) f.close() db_info = None for db_info_entry in data['dbs']: if db_info_entry['name'] == db_name: db_info = db_info_entry break if db_info == None: print "Invalid db name[{db_name}]".format(**locals()) return dbSettings = {} dbSettings["host"]=re.search('://(.*?):(.*?),',db_info["jdbc"]).groups()[0] dbSettings["user"]=db_info["user"] dbSettings["password"]=db_info["password"] seldon_utils.addApiDb(db_name, dbSettings) seldon_utils.addClientDb(client_name, dbSettings, consumer_details=None) # write to local file data_fpath = get_data_fpath(zkroot, client_name) data = {'DB_JNDI_NAME': db_name} write_data_to_file(data_fpath, data) # write to zookeeper node_path=gdata["all_clients_node_path"]+"/"+client_name data_json = dict_to_json(data) zk_utils.node_set(zk_client, node_path, data_json)
def action_configure(command_data, opts): client_name = opts.client_name if (client_name == None) or (len(client_name) == 0): print "Need client name to comfigure explanations for" sys.exit(1) zkroot = command_data["zkdetails"]["zkroot"] if not is_existing_client(zkroot, client_name): print "Invalid client[{client_name}]".format(**locals()) sys.exit(1) zk_client = command_data["zkdetails"]["zk_client"] ensure_client_has_recommendation_explanation(zkroot, zk_client, client_name) data_fpath = zkroot + gdata["all_clients_node_path"] + "/" + client_name + "/recommendation_explanation/_data_" f = open(data_fpath) json = f.read() f.close() data = seldon_utils.json_to_dict(json) if opts.cache_enabled != None: data["cache_enabled"] = True if opts.cache_enabled == "true" else False if opts.default_locale!= None: data["default_locale"] = opts.default_locale if opts.explanations_enabled != None: data["explanations_enabled"] = True if opts.explanations_enabled == "true" else False write_data_to_file(data_fpath, data) node_path = gdata["all_clients_node_path"]+"/"+client_name+"/recommendation_explanation" zk_utils.node_set(zk_client, node_path, seldon_utils.dict_to_json(data)) action_show(command_data, opts)
def action_commit(command_data, opts): client_name = opts.client_name if client_name == None: print "Need client name to add algs for" sys.exit(1) zkroot = command_data["zkdetails"]["zkroot"] if not is_existing_client(zkroot, client_name): print "Invalid client[{client_name}]".format(**locals()) return zk_client = command_data["zkdetails"]["zk_client"] zkroot = command_data["zkdetails"]["zkroot"] data_fpath = zkroot + gdata[ "all_clients_node_path"] + "/" + client_name + "/predict_algs/_data_" if not os.path.isfile(data_fpath): "Data to commit not found!!" f = open(data_fpath) data_json = f.read() f.close() zk_client = command_data["zkdetails"]["zk_client"] node_path = gdata[ "all_clients_node_path"] + "/" + client_name + "/predict_algs" zk_utils.node_set(zk_client, node_path, data_json)
def action_edit(command_data, opts): client_name = opts.client_name if client_name == None: print "Need client name to show the attr for" sys.exit(1) zkroot = command_data["zkdetails"]["zkroot"] if not is_existing_client(zkroot, client_name): print "Invalid client[{client_name}]".format(**locals()) sys.exit(1) zk_client = command_data["zkdetails"]["zk_client"] ensure_client_has_attr(zkroot, zk_client, client_name) data_fpath = zkroot + gdata["all_clients_node_path"] + "/" + client_name + "/attr/_data_" #do the edit from subprocess import call editor=seldon_utils.get_editor() call([editor, data_fpath]) f = open(data_fpath) json = f.read() f.close() data = seldon_utils.json_to_dict(json) if data is None: print "Invalid attr json!" else: write_data_to_file(data_fpath, data) node_path = gdata["all_clients_node_path"]+"/"+client_name+"/attr" zk_utils.node_set(zk_client, node_path, seldon_utils.dict_to_json(data)) show_attr(data)
def subcmd_add(command_data): client_name = command_data["subcmd_args"][0] if len( command_data["subcmd_args"]) > 0 else None if client_name == None: print "Need client name to add model for" return zkroot = command_data["zkdetails"]["zkroot"] if not is_existing_client(zkroot, client_name): print "Invalid client[{client_name}]".format(**locals()) return default_models = command_data["default_models"] models = default_models.keys() for idx, model in enumerate(models): print " [{idx}] {model}".format(**locals()) q = "Choose model: " model_idx = None try: model_idx = int(raw_input(q)) except ValueError: pass model_name = models[ model_idx] if model_idx != None and model_idx >= 0 and model_idx < len( models) else None if model_name == None: print "Invaild model" return data_fpath = "{zkroot}{all_clients_node_path}/{client_name}/offline/{model_name}/_data_".format( zkroot=zkroot, all_clients_node_path=gdata["all_clients_node_path"], client_name=client_name, model_name=model_name) zk_client = command_data["zkdetails"]["zk_client"] if not os.path.isfile(data_fpath): node_path = "{all_clients_node_path}/{client_name}/offline/{model_name}".format( all_clients_node_path=gdata["all_clients_node_path"], client_name=client_name, model_name=model_name) if zk_client.exists(node_path): write_node_value_to_file(zk_client, zkroot, node_path) else: default_model_data = default_models[model_name]["config"] if default_model_data.has_key("inputPath"): default_model_data["inputPath"] = command_data["seldon_models"] if default_model_data.has_key("outputPath"): default_model_data["outputPath"] = command_data[ "seldon_models"] data = default_model_data write_data_to_file(data_fpath, data) zk_utils.node_set(zk_client, node_path, seldon_utils.dict_to_json(data)) else: print "Model [{model_name}] already added".format(**locals())
def subcmd_edit(command_data): def have_client_name(): if client_name == None: print "Need client name to edit model for" return False else: return True def have_valid_client_name(): if not is_existing_client(zkroot, client_name): print "Invalid client[{client_name}]".format(**locals()) return False else: return True client_name = command_data["subcmd_args"][0] if len(command_data["subcmd_args"])>0 else None if not have_client_name(): return zk_client = command_data["zkdetails"]["zk_client"] zkroot = command_data["zkdetails"]["zkroot"] if not have_valid_client_name(): return models_for_client_fpath = "{zkroot}{all_clients_node_path}/{client_name}/offline".format(zkroot=zkroot,all_clients_node_path=gdata["all_clients_node_path"],client_name=client_name) models = os.listdir(models_for_client_fpath) for idx,model in enumerate(models): print " [{idx}] {model}".format(**locals()) q="Choose model: " model_idx = None try: model_idx = int(raw_input(q)) except ValueError: pass model_name = models[model_idx] if model_idx != None and model_idx>=0 and model_idx<len(models) else None if model_name == None: print "Invaild model" return data_fpath = "{zkroot}{all_clients_node_path}/{client_name}/offline/{model_name}/_data_".format(zkroot=zkroot,all_clients_node_path=gdata["all_clients_node_path"],client_name=client_name,model_name=model_name) #do the edit editor=seldon_utils.get_editor() call([editor, data_fpath]) f = open(data_fpath) json = f.read() f.close() data = seldon_utils.json_to_dict(json) if data is None: print "Invalid model json!" else: write_data_to_file(data_fpath, data) node_path = "{all_clients_node_path}/{client_name}/offline/{model_name}".format(all_clients_node_path=gdata["all_clients_node_path"],client_name=client_name,model_name=model_name) pp(node_path) zk_utils.node_set(zk_client, node_path, seldon_utils.dict_to_json(data))
def action_edit(command_data, opts, extra_args): zkroot = command_data["zkdetails"]["zkroot"] def get_valid_client(): client_name = opts.client_name if client_name == None: print "Need client name to show models for" sys.exit(1) if not is_existing_client(zkroot, client_name): print "Invalid client[{client_name}]".format(**locals()) sys.exit(1) return client_name def get_valid_model(): model_name = opts.model_name if model_name == None: print "Need model name to use" sys.exit(1) default_models = command_data["conf_data"]["default_models"] if model_name not in default_models.keys(): print "Invalid model name: {model_name}".format(**locals()) sys.exit(1) return model_name client_name = get_valid_client() model_name = get_valid_model() zk_client = command_data["zkdetails"]["zk_client"] data_fpath = "{zkroot}{all_clients_node_path}/{client_name}/offline/{model_name}/_data_".format( zkroot=zkroot, all_clients_node_path=gdata["all_clients_node_path"], client_name=client_name, model_name=model_name) #do the edit editor = seldon_utils.get_editor() call([editor, data_fpath]) f = open(data_fpath) json = f.read() f.close() data = seldon_utils.json_to_dict(json) if data is None: print "Invalid model json!" else: write_data_to_file(data_fpath, data) node_path = "{all_clients_node_path}/{client_name}/offline/{model_name}".format( all_clients_node_path=gdata["all_clients_node_path"], client_name=client_name, model_name=model_name) pp(node_path) zk_utils.node_set(zk_client, node_path, seldon_utils.dict_to_json(data))
def ensure_client_has_algs(zk_client): node_path = gdata["all_clients_node_path"] + "/" + client_name + "/algs" if zk_client.exists(node_path): write_node_value_to_file(zk_client, zkroot, node_path) else: default_alg_json = '{"algorithms":[{"config":[],"filters":[],"includers":[],"name":"recentItemsRecommender"}],"combiner":"firstSuccessfulCombiner"}' data = json_to_dict(default_alg_json) write_data_to_file(data_fpath, data) zk_utils.node_set(zk_client, node_path, dict_to_json(data))
def ensure_client_has_algs(zkroot, zk_client, client_name): data_fpath = "{zkroot}{all_clients_node_path}/{client_name}/predict_algs/_data_".format(zkroot=zkroot,all_clients_node_path=gdata["all_clients_node_path"],client_name=client_name) if not os.path.isfile(data_fpath): node_path = gdata["all_clients_node_path"]+"/"+client_name+"/predict_algs" if zk_client.exists(node_path): write_node_value_to_file(zk_client, zkroot, node_path) else: default_alg_json = '{"algorithms":[]}' data = json_to_dict(default_alg_json) write_data_to_file(data_fpath, data) zk_utils.node_set(zk_client, node_path, dict_to_json(data))
def action_set(command_data, opts): if "grpc_util_home" in command_data["conf_data"]: grpc_home = command_data["conf_data"]["grpc_util_home"] else: print "Need the location for grpc_util_home to create proto jar via maven" sys.exit(1) client_name = opts.client_name if client_name == None: print "Need client name to show the rpc settings for" sys.exit(1) proto_file = opts.proto if proto_file == None: print "Need --proto argument" sys.exit(1) request_class = opts.request_class if request_class == None: print "Need request class to setup!" sys.exit(1) jar_file = "/seldon-data/rpc/jar/" + client_name + ".jar" if not create_jar(grpc_home, opts.proto, jar_file) == 0: print "Failed to create jar file from proto" sys.exit(1) zkroot = command_data["zkdetails"]["zkroot"] zk_client = command_data["zkdetails"]["zk_client"] if opts.reply_class == None: data = { "jarFilename": jar_file, "requestClassName": opts.request_class } else: data = { "jarFilename": jar_file, "requestClassName": opts.request_class, "replyClassName": opts.reply_class } data_fpath = zkroot + gdata[ "all_clients_node_path"] + "/" + client_name + "/rpc/_data_" node_path = gdata["all_clients_node_path"] + "/" + client_name + "/rpc" write_data_to_file(data_fpath, data) f = open(data_fpath) data_json = f.read() f.close() zk_client = command_data["zkdetails"]["zk_client"] zk_utils.node_set(zk_client, node_path, data_json)
def ensure_client_has_recommendation_explanation(zkroot, zk_client, client_name): data_fpath = "{zkroot}{all_clients_node_path}/{client_name}/recommendation_explanation/_data_".format(zkroot=zkroot,all_clients_node_path=gdata["all_clients_node_path"],client_name=client_name) if not os.path.isfile(data_fpath): node_path = gdata["all_clients_node_path"]+"/"+client_name+"/recommendation_explanation" if zk_client.exists(node_path): write_node_value_to_file(zk_client, zkroot, node_path) else: default_rec_exp = '{"cache_enabled":true,"default_locale":"us-en","explanations_enabled":false}' data = seldon_utils.json_to_dict(default_rec_exp) write_data_to_file(data_fpath, data) zk_utils.node_set(zk_client, node_path, seldon_utils.dict_to_json(data))
def ensure_client_has_algs(zkroot, zk_client, client_name): data_fpath = "{zkroot}{all_clients_node_path}/{client_name}/algs/_data_".format(zkroot=zkroot,all_clients_node_path=gdata["all_clients_node_path"],client_name=client_name) if not os.path.isfile(data_fpath): node_path = gdata["all_clients_node_path"]+"/"+client_name+"/algs" if zk_client.exists(node_path): write_node_value_to_file(zk_client, zkroot, node_path) else: default_alg_json = '{"algorithms":[{"config":[],"filters":[],"includers":[],"name":"recentItemsRecommender"}],"combiner":"firstSuccessfulCombiner"}' data = json_to_dict(default_alg_json) write_data_to_file(data_fpath, data) zk_utils.node_set(zk_client, node_path, dict_to_json(data))
def action_add(command_data, opts, extra_args): client_name = opts.client_name if client_name == None: print "Need client name to add model for" sys.exit(1) zkroot = command_data["zkdetails"]["zkroot"] if not is_existing_client(zkroot, client_name): print "Invalid client[{client_name}]".format(**locals()) sys.exit(1) model_name = opts.model_name if model_name == None: print "Need model name to use" sys.exit(1) default_models = command_data["conf_data"]["default_models"] if model_name not in default_models.keys(): print "Invalid model name: {model_name}".format(**locals()) sys.exit(1) config_args = get_config_args(extra_args) data_fpath = "{zkroot}{all_clients_node_path}/{client_name}/offline/{model_name}/_data_".format(zkroot=zkroot,all_clients_node_path=gdata["all_clients_node_path"],client_name=client_name,model_name=model_name) zk_client = command_data["zkdetails"]["zk_client"] node_path = "{all_clients_node_path}/{client_name}/offline/{model_name}".format(all_clients_node_path=gdata["all_clients_node_path"],client_name=client_name,model_name=model_name) if not os.path.isfile(data_fpath): if zk_client.exists(node_path): write_node_value_to_file(zk_client, zkroot, node_path) f = open(data_fpath) json = f.read() f.close() data = seldon_utils.json_to_dict(json) else: default_model_data = default_models[model_name]["config"] if default_model_data.has_key("inputPath"): default_model_data["inputPath"]=command_data["conf_data"]["seldon_models"] if default_model_data.has_key("outputPath"): default_model_data["outputPath"]=command_data["conf_data"]["seldon_models"] data = default_model_data else: f = open(data_fpath) json = f.read() f.close() data = seldon_utils.json_to_dict(json) print "Model [{model_name}] already added".format(**locals()) for k in config_args: print "adding config ",k,":",config_args[k] data[k] = config_args[k] write_data_to_file(data_fpath, data) zk_utils.node_set(zk_client, node_path, seldon_utils.dict_to_json(data))
def action_commit(command_data, opts): zkroot = command_data["zkdetails"]["zkroot"] data_fpath = zkroot + gdata['data_path'] if not os.path.isfile(data_fpath): write_data_to_file(data_fpath, gdata["default_data"]) f = open(data_fpath) data_json = f.read() f.close() zk_client=command_data["zkdetails"]["zk_client"] node_path=gdata["node_path"] zk_utils.node_set(zk_client, node_path, data_json)
def ensure_client_has_attr(zkroot, zk_client, client_name): data_fpath = "{zkroot}{all_clients_node_path}/{client_name}/attr/_data_".format( zkroot=zkroot,all_clients_node_path=gdata["all_clients_node_path"],client_name=client_name) if not os.path.isfile(data_fpath): node_path = gdata["all_clients_node_path"]+"/"+client_name+"/attr" if zk_client.exists(node_path): write_node_value_to_file(zk_client, zkroot, node_path) else: default_attr_json = '{"types":[{"type_attrs":[{"name":"title","value_type":"string"}],"type_id":1,"type_name":"defaulttype"}]}' data = seldon_utils.json_to_dict(default_attr_json) write_data_to_file(data_fpath, data) zk_utils.node_set(zk_client, node_path, seldon_utils.dict_to_json(data))
def action_commit(command_data, opts): zkroot = command_data["zkdetails"]["zkroot"] data_fpath = zkroot + gdata['data_path'] if not os.path.isfile(data_fpath): write_data_to_file(data_fpath, gdata["default_data"]) f = open(data_fpath) data_json = f.read() f.close() zk_client = command_data["zkdetails"]["zk_client"] node_path = gdata["node_path"] zk_utils.node_set(zk_client, node_path, data_json)
def subcmd_commit(command_data): zkroot = command_data["zkdetails"]["zkroot"] zk_client = command_data["zkdetails"]["zk_client"] data_fpath = zkroot + gdata['data_path'] ensure_local_data_file_exists(zk_client, zkroot) f = open(data_fpath) data_json = f.read() f.close() zk_client=command_data["zkdetails"]["zk_client"] node_path=gdata["node_path"] zk_utils.node_set(zk_client, node_path, data_json)
def subcmd_commit(command_data): zkroot = command_data["zkdetails"]["zkroot"] zk_client = command_data["zkdetails"]["zk_client"] data_fpath = zkroot + gdata['data_path'] ensure_local_data_file_exists(zk_client, zkroot) f = open(data_fpath) data_json = f.read() f.close() zk_client = command_data["zkdetails"]["zk_client"] node_path = gdata["node_path"] zk_utils.node_set(zk_client, node_path, data_json)
def action_edit(command_data, opts): zkroot = command_data["zkdetails"]["zkroot"] def get_valid_client(): client_name = opts.client_name if client_name == None: print "Need client name to show models for" sys.exit(1) if not is_existing_client(zkroot, client_name): print "Invalid client[{client_name}]".format(**locals()) sys.exit(1) return client_name def get_valid_model(): model_name = opts.model_name if model_name == None: print "Need model name to use" sys.exit(1) default_models = command_data["conf_data"]["default_models"] if model_name not in default_models.keys(): print "Invalid model name: {model_name}".format(**locals()) sys.exit(1) return model_name client_name = get_valid_client() model_name = get_valid_model() zk_client = command_data["zkdetails"]["zk_client"] data_fpath = "{zkroot}{all_clients_node_path}/{client_name}/offline/{model_name}/_data_".format(zkroot=zkroot,all_clients_node_path=gdata["all_clients_node_path"],client_name=client_name,model_name=model_name) #do the edit editor=seldon_utils.get_editor() call([editor, data_fpath]) f = open(data_fpath) json = f.read() f.close() data = seldon_utils.json_to_dict(json) if data is None: print "Invalid model json!" else: write_data_to_file(data_fpath, data) node_path = "{all_clients_node_path}/{client_name}/offline/{model_name}".format(all_clients_node_path=gdata["all_clients_node_path"],client_name=client_name,model_name=model_name) pp(node_path) zk_utils.node_set(zk_client, node_path, seldon_utils.dict_to_json(data))
def action_set(command_data, opts): if "grpc_util_home" in command_data["conf_data"]: grpc_home = command_data["conf_data"]["grpc_util_home"] else: print "Need the location for grpc_util_home to create proto jar via maven" sys.exit(1) client_name = opts.client_name if client_name == None: print "Need client name to show the rpc settings for" sys.exit(1) proto_file = opts.proto if proto_file == None: print "Need --proto argument" sys.exit(1) request_class = opts.request_class if request_class == None: print "Need request class to setup!" sys.exit(1) jar_file = "/seldon-data/rpc/jar/"+client_name+".jar" if not create_jar(grpc_home,opts.proto,jar_file) == 0: print "Failed to create jar file from proto" sys.exit(1) zkroot = command_data["zkdetails"]["zkroot"] zk_client = command_data["zkdetails"]["zk_client"] if opts.reply_class == None: data = {"jarFilename":jar_file,"requestClassName":opts.request_class} else: data = {"jarFilename":jar_file,"requestClassName":opts.request_class,"replyClassName":opts.reply_class} data_fpath = zkroot + gdata["all_clients_node_path"] + "/" + client_name + "/rpc/_data_" node_path = gdata["all_clients_node_path"] + "/" + client_name + "/rpc" write_data_to_file(data_fpath, data) f = open(data_fpath) data_json = f.read() f.close() zk_client=command_data["zkdetails"]["zk_client"] zk_utils.node_set(zk_client, node_path, data_json)
def ensure_client_has_recommendation_explanation(zkroot, zk_client, client_name): data_fpath = "{zkroot}{all_clients_node_path}/{client_name}/recommendation_explanation/_data_".format( zkroot=zkroot, all_clients_node_path=gdata["all_clients_node_path"], client_name=client_name) if not os.path.isfile(data_fpath): node_path = gdata[ "all_clients_node_path"] + "/" + client_name + "/recommendation_explanation" if zk_client.exists(node_path): write_node_value_to_file(zk_client, zkroot, node_path) else: default_rec_exp = '{"cache_enabled":true,"default_locale":"us-en","explanations_enabled":false}' data = seldon_utils.json_to_dict(default_rec_exp) write_data_to_file(data_fpath, data) zk_utils.node_set(zk_client, node_path, seldon_utils.dict_to_json(data))
def store_json(command_data,opts): f = open(opts.json) json = f.read() f.close() data = seldon_utils.json_to_dict(json) zk_client = command_data["zkdetails"]["zk_client"] zkroot = command_data["zkdetails"]["zkroot"] data_fpath = zkroot + gdata["all_clients_node_path"] + "/" + opts.client_name + "/attr/_data_" if data is None: print "Invalid attr json!" sys.exit(1) else: write_data_to_file(data_fpath, data) node_path = gdata["all_clients_node_path"]+"/"+opts.client_name+"/attr" zk_utils.node_set(zk_client, node_path, seldon_utils.dict_to_json(data))
def subcmd_add(command_data): client_name = command_data["subcmd_args"][0] if len(command_data["subcmd_args"])>0 else None if client_name == None: print "Need client name to add model for" return zkroot = command_data["zkdetails"]["zkroot"] if not is_existing_client(zkroot, client_name): print "Invalid client[{client_name}]".format(**locals()) return default_models = command_data["default_models"] models = default_models.keys() for idx,model in enumerate(models): print " [{idx}] {model}".format(**locals()) q="Choose model: " model_idx = None try: model_idx = int(raw_input(q)) except ValueError: pass model_name = models[model_idx] if model_idx != None and model_idx>=0 and model_idx<len(models) else None if model_name == None: print "Invaild model" return data_fpath = "{zkroot}{all_clients_node_path}/{client_name}/offline/{model_name}/_data_".format(zkroot=zkroot,all_clients_node_path=gdata["all_clients_node_path"],client_name=client_name,model_name=model_name) zk_client = command_data["zkdetails"]["zk_client"] if not os.path.isfile(data_fpath): node_path = "{all_clients_node_path}/{client_name}/offline/{model_name}".format(all_clients_node_path=gdata["all_clients_node_path"],client_name=client_name,model_name=model_name) if zk_client.exists(node_path): write_node_value_to_file(zk_client, zkroot, node_path) else: default_model_data = default_models[model_name]["config"] if default_model_data.has_key("inputPath"): default_model_data["inputPath"]=command_data["seldon_models"] if default_model_data.has_key("outputPath"): default_model_data["outputPath"]=command_data["seldon_models"] data = default_model_data write_data_to_file(data_fpath, data) zk_utils.node_set(zk_client, node_path, seldon_utils.dict_to_json(data)) else: print "Model [{model_name}] already added".format(**locals())
def action_set(command_data, opts): client_name = opts.client_name if client_name == None: print "Need client name to show the rpc settings for" sys.exit(1) jar_file = opts.jar if jar_file == None: print "Need protobuf jar file location to setup!" sys.exit(1) request_class = opts.request_class if request_class == None: print "Need request class to setup!" sys.exit(1) zkroot = command_data["zkdetails"]["zkroot"] zk_client = command_data["zkdetails"]["zk_client"] if opts.reply_class == None: data = { "jarFilename": opts.jar, "requestClassName": opts.request_class } else: data = { "jarFilename": opts.jar, "requestClassName": opts.request_class, "replyClassName": opts.reply_class } data_fpath = zkroot + gdata[ "all_clients_node_path"] + "/" + client_name + "/rpc/_data_" node_path = gdata["all_clients_node_path"] + "/" + client_name + "/rpc" write_data_to_file(data_fpath, data) f = open(data_fpath) data_json = f.read() f.close() zk_client = command_data["zkdetails"]["zk_client"] zk_utils.node_set(zk_client, node_path, data_json)
def action_commit(command_data, opts): client_name = opts.client_name if client_name == None: print "Need client name to add algs for" sys.exit(1) zkroot = command_data["zkdetails"]["zkroot"] if not is_existing_client(zkroot, client_name): print "Invalid client[{client_name}]".format(**locals()) return zk_client = command_data["zkdetails"]["zk_client"] zkroot = command_data["zkdetails"]["zkroot"] data_fpath = zkroot + gdata[ "all_clients_node_path"] + "/" + client_name + "/algs/_data_" if not os.path.isfile(data_fpath): "Data to commit not found!!" f = open(data_fpath) data_json = f.read() f.close() zk_client = command_data["zkdetails"]["zk_client"] node_path = gdata["all_clients_node_path"] + "/" + client_name + "/algs" zk_utils.node_set(zk_client, node_path, data_json) # activate any required models default_algorithms = command_data["conf_data"]["default_algorithms"] data = json_to_dict(data_json) algorithms = data["algorithms"] print "algorithms:" for alg in algorithms: alg_name = alg["name"] if default_algorithms[alg_name].has_key("zk_activate_node"): node_path = default_algorithms[alg_name]["zk_activate_node"] node_fpath = zkroot + node_path + "/_data_" data_models = get_data_from_file(node_fpath) zk_utils.node_set(zk_client, node_path, data_models)
def action_commit(command_data, opts): client_name = opts.client_name if client_name == None: print "Need client name to add algs for" sys.exit(1) zkroot = command_data["zkdetails"]["zkroot"] if not is_existing_client(zkroot, client_name): print "Invalid client[{client_name}]".format(**locals()) return zk_client = command_data["zkdetails"]["zk_client"] zkroot = command_data["zkdetails"]["zkroot"] data_fpath = zkroot + gdata["all_clients_node_path"] + "/" + client_name + "/predict_algs/_data_" if not os.path.isfile(data_fpath): "Data to commit not found!!" f = open(data_fpath) data_json = f.read() f.close() zk_client = command_data["zkdetails"]["zk_client"] node_path = gdata["all_clients_node_path"] + "/" + client_name + "/predict_algs" zk_utils.node_set(zk_client, node_path, data_json)
def subcmd_commit(command_data): client_name = command_data["subcmd_args"][0] if len(command_data["subcmd_args"])>0 else None if client_name == None: print "Need client name to delete algs for" return zkroot = command_data["zkdetails"]["zkroot"] if not is_existing_client(zkroot, client_name): print "Invalid client[{client_name}]".format(**locals()) return zk_client = command_data["zkdetails"]["zk_client"] zkroot = command_data["zkdetails"]["zkroot"] data_fpath = zkroot + gdata["all_clients_node_path"] + "/" + client_name + "/algs/_data_" if not os.path.isfile(data_fpath): "Data to commit not found!!" f = open(data_fpath) data_json = f.read() f.close() zk_client = command_data["zkdetails"]["zk_client"] node_path = gdata["all_clients_node_path"] + "/" + client_name + "/algs" zk_utils.node_set(zk_client, node_path, data_json)
def action_commit(command_data, opts): client_name = opts.client_name if client_name == None: print "Need client name to add algs for" sys.exit(1) zkroot = command_data["zkdetails"]["zkroot"] if not is_existing_client(zkroot, client_name): print "Invalid client[{client_name}]".format(**locals()) return zk_client = command_data["zkdetails"]["zk_client"] zkroot = command_data["zkdetails"]["zkroot"] data_fpath = zkroot + gdata["all_clients_node_path"] + "/" + client_name + "/algs/_data_" if not os.path.isfile(data_fpath): "Data to commit not found!!" f = open(data_fpath) data_json = f.read() f.close() zk_client = command_data["zkdetails"]["zk_client"] node_path = gdata["all_clients_node_path"] + "/" + client_name + "/algs" zk_utils.node_set(zk_client, node_path, data_json) # activate any required models default_algorithms = command_data["conf_data"]["default_algorithms"] data = json_to_dict(data_json) algorithms = data["algorithms"] print "algorithms:" for alg in algorithms: alg_name=alg["name"] if default_algorithms[alg_name].has_key("zk_activate_node"): node_path = default_algorithms[alg_name]["zk_activate_node"] node_fpath = zkroot + node_path + "/_data_" data_models = get_data_from_file(node_fpath) zk_utils.node_set(zk_client, node_path, data_models)
def action_commit(command_data, opts): client_name = opts.client_name if client_name == None: print "Need client name to commit data for" sys.exit(1) zkroot = command_data["zkdetails"]["zkroot"] if not is_existing_client(zkroot, client_name): print "Invalid client[{client_name}]".format(**locals()) return zk_client = command_data["zkdetails"]["zk_client"] zkroot = command_data["zkdetails"]["zkroot"] data_fpath = zkroot + gdata[ "all_clients_node_path"] + "/" + client_name + "/alg_rectags/_data_" if os.path.isfile(data_fpath): f = open(data_fpath) data_json = f.read() f.close() zk_client = command_data["zkdetails"]["zk_client"] node_path = gdata[ "all_clients_node_path"] + "/" + client_name + "/alg_rectags" zk_utils.node_set(zk_client, node_path, data_json) # activate any required models data = json_to_dict(data_json) recommender_set = set() if data["defaultStrategy"].has_key("algorithms"): for alg in data["defaultStrategy"]["algorithms"]: recommender_set.add(alg["name"]) elif data["defaultStrategy"].has_key("variations"): for variation in data["defaultStrategy"]["variations"]: for alg in variation["config"]["algorithms"]: recommender_set.add(alg["name"]) default_algorithms = command_data["conf_data"]["default_algorithms"] for recommender_name in recommender_set: if default_algorithms[recommender_name].has_key( "zk_activate_node"): print "activate", recommender_name node_path = default_algorithms[recommender_name][ "zk_activate_node"] node_fpath = zkroot + node_path + "/_data_" data_models = get_data_from_file(node_fpath) zk_utils.node_set(zk_client, node_path, data_models) return #TODO remove the following once only using alg_rectags data_fpath = zkroot + gdata[ "all_clients_node_path"] + "/" + client_name + "/algs/_data_" if not os.path.isfile(data_fpath): print "Data to commit not found!!" sys.exit(1) f = open(data_fpath) data_json = f.read() f.close() zk_client = command_data["zkdetails"]["zk_client"] node_path = gdata["all_clients_node_path"] + "/" + client_name + "/algs" zk_utils.node_set(zk_client, node_path, data_json) # activate any required models default_algorithms = command_data["conf_data"]["default_algorithms"] data = json_to_dict(data_json) algorithms = data["algorithms"] print "algorithms:" for alg in algorithms: alg_name = alg["name"] if default_algorithms[alg_name].has_key("zk_activate_node"): node_path = default_algorithms[alg_name]["zk_activate_node"] node_fpath = zkroot + node_path + "/_data_" data_models = get_data_from_file(node_fpath) zk_utils.node_set(zk_client, node_path, data_models)
def action_add(command_data, opts, extra_args): client_name = opts.client_name if client_name == None: print "Need client name to add model for" sys.exit(1) zkroot = command_data["zkdetails"]["zkroot"] if not is_existing_client(zkroot, client_name): print "Invalid client[{client_name}]".format(**locals()) sys.exit(1) model_name = opts.model_name if model_name == None: print "Need model name to use" sys.exit(1) default_models = command_data["conf_data"]["default_models"] if model_name not in default_models.keys(): print "Invalid model name: {model_name}".format(**locals()) sys.exit(1) config_args = get_config_args(extra_args) data_fpath = "{zkroot}{all_clients_node_path}/{client_name}/offline/{model_name}/_data_".format( zkroot=zkroot, all_clients_node_path=gdata["all_clients_node_path"], client_name=client_name, model_name=model_name) zk_client = command_data["zkdetails"]["zk_client"] node_path = "{all_clients_node_path}/{client_name}/offline/{model_name}".format( all_clients_node_path=gdata["all_clients_node_path"], client_name=client_name, model_name=model_name) if not os.path.isfile(data_fpath): if zk_client.exists(node_path): write_node_value_to_file(zk_client, zkroot, node_path) f = open(data_fpath) json = f.read() f.close() data = seldon_utils.json_to_dict(json) else: default_model_data = default_models[model_name]["config"] if default_model_data.has_key("inputPath"): default_model_data["inputPath"] = command_data["conf_data"][ "seldon_models"] if default_model_data.has_key("outputPath"): default_model_data["outputPath"] = command_data["conf_data"][ "seldon_models"] data = default_model_data else: f = open(data_fpath) json = f.read() f.close() data = seldon_utils.json_to_dict(json) print "Model [{model_name}] already added".format(**locals()) for k in config_args: print "adding config ", k, ":", config_args[k] data[k] = config_args[k] write_data_to_file(data_fpath, data) zk_utils.node_set(zk_client, node_path, seldon_utils.dict_to_json(data))