def get(self, request, nnid, type): """ - desc : return network data format information """ #get_network_config try: result_temp = netconf.get_network_config(nnid) datadesc = netconf.load_ori_format(nnid) result_datadesc_source = json.loads(datadesc) result = dict() if type == "cell_feature": result = result_datadesc_source["cell_feature"] elif type == "label": result = result_datadesc_source["label"] elif type == "all": result = result_datadesc_source["cell_feature"] result.update(result_datadesc_source["label"]) elif type == "labels": result = data.ImageManager().get_label_list(nnid) return_data = {"status": "200", "result": result} return Response(json.dumps(return_data)) except Exception as e: return_data = {"status": "400", "result": str(e)} return Response(json.dumps(return_data))
def get_json_by_nnid(self,nnid): """get network configuration info json :param nnid :return: json string """ datadesc = netconf.load_ori_format(nnid) result = json.loads(datadesc) utils.tfmsa_logger("((Wdnn_common_model_get_json_by_nnid)) ## END##") return result
def get(self, request, baseid, table, nnid): """ - desc : return network data format information """ try: result = netconf.load_ori_format(nnid) return_data = {"status": "200", "result": json.loads(result)} return Response(json.dumps(return_data)) except Exception as e: return_data = {"status": "400", "result": str(e)} return Response(json.dumps(return_data))
def get_init_info_wdnn(self, nnid): """ Get infomation of Wdnn initial :param nnid :param model_dir : directory of chkpoint of wdnn model """ json_string = netconf.load_ori_format(nnid) json_object = json.loads(json_string) utils.tfmsa_logger("[4]load net conf form db") conf = netconf.load_conf(nnid) hidden_layers_value = conf.layer result_temp = netconf.get_network_config(nnid) label_cnt = json.loads(json.dumps(result_temp)) label_object = label_cnt["datasets"] utils.tfmsa_logger("((1.Make WDN Network Build)) set up Hidden Layers (" + str(hidden_layers_value) + ")") return conf, hidden_layers_value, json_object, label_object
def wdd_eval(self, nnid, filename=None): """ Wide & Deep Network predict :param nnid : network id in tfmsacore_nninfo :return: acturacy """ try: utils.tfmsa_logger("((3.1 Wide & Deep Network Eval Start)) ## " + nnid + "##") json_string = WdnnCommonManager.get_all_info_json_by_nnid( self, nnid=nnid) database = json_string['dir'] table_name = json_string['table'] json_string_desc = netconf.load_ori_format(nnid) json_ob = json.loads(json_string_desc) #get datadesc format model_dir = str(json_string['query']) label_column = list(json_ob["label"].keys())[0] _label_list = json_string['datasets'] utils.tfmsa_logger( "((3.1 Wide & Deep Network Eval Start)) ## DB : " + database + " TABLE : " + table_name) utils.tfmsa_logger( "((3.1 Wide & Deep Network Eval Start)) ## Model_Dir : " + model_dir) utils.tfmsa_logger( "((3.1 Wide & Deep Network Eval Start)) ## Label Column " + label_column + " Label : " + _label_list) label_list = eval(_label_list) wdnn_model = WdnnCommonManager.wdnn_build(self, nnid, model_dir, False) if filename == None: df, last_key = data.DataMaster().query_data( "test_schema_" + database, table_name, "a", use_df=True, limit_cnt=-1, with_label=label_column) utils.tfmsa_logger( "((3.1 Wide & Deep Network Eval Start)) ## Get Test Schema Data Count" + str(len(df)) + "LastKey : " + str(last_key)) else: #Don't use this pat print("((2.Get Dataframe from CSV)) ##Start## (" + nnid + " , " + filename + ")") file_path = settings.FILE_ROOT + "/predict/" + nnid + "/" + filename print("((2.Get Dataframe from CSV)) ##filePath## (" + file_path + ")") print(file_path) df = pd.read_csv( tf.gfile.Open(file_path), # names=COLUMNS, skipinitialspace=True, engine="python") df['label'] = ( df[label_column].apply(lambda x: "Y" in x)).astype(int) results = {} utils.tfmsa_logger( "((3.1 Wide & Deep Network Eval Start)) ## Start Predict in Eval Method ##" ) predict_results = wdnn_model.predict( input_fn=lambda: WdnnCommonManager.input_fn(self, df, nnid)) df['predict_label'] = list(predict_results) #print(df['predict_label'] ) utils.tfmsa_logger( "make label encorder function ######################") # #Decode Label(int) to Label(str) le = LabelEncoder() le.fit(label_list) lable_decoder_func = lambda x: le.inverse_transform([x]) df['predict_label'] = df['predict_label'].map( lable_decoder_func).astype("str") label_value = list(df['label'].unique()) label_encode = le.inverse_transform(label_value) utils.tfmsa_logger( "((3.1 Wide & Deep Network Predict)) ## Label Convert ##" + str(label_value) + "DECODE ---> " + str(label_encode)) #results_acc table manager class create acc_result_obj = AccStaticResult() log_count = 0 for value in df.iterrows(): log_count += 1 #print("Inside start row by row") #print(type(value)) _row = value[1] ori_label = str(_row[label_column]) predict_label = str(eval(_row["predict_label"]) [0]) #str(list(_row["predict_label"])[0]) #predict_label = le.inverse_transform(_predict_label) if (log_count % 20 == 0): utils.tfmsa_logger( "((3.1 Wide & Deep Network Eval Columns Insert)) " + str(ori_label) + "------->" + str(predict_label)) acc_result_obj = AccEvalCommon(nnid).set_result( acc_result_obj, ori_label, predict_label) return results except Exception as e: print("Error Message : {0}".format(e)) raise Exception(e)
def put_data(self, data_frame, table_name, label, file_set, nnid): """ delete label folder under table :param db_name: :param table_name: :return: """ try: tfmsa_logger("[1]Start upload images...") self.make_inital_path(nnid) # get network base info tfmsa_logger("[2]get network base info") net_info = netconf.get_network_config(nnid) # get data format info tfmsa_logger("[3]get network format info") format_info = json.loads(netconf.load_ori_format(nnid)) # get hbase trasaction table tfmsa_logger("[4]get hbase trasaction table") conn, train_table, test_table = self.get_divided_target_table( data_frame, table_name) train_buffer = train_table.batch(transaction=True) test_buffer = test_table.batch(transaction=True) #get Label list tfmsa_logger("[5]Updata Label List ") self.label_info_update(net_info, label) # get Label list tfmsa_logger("[6]upload image on Hbase - start ") file_list = [] train_key_set, test_key_set = self.divide_train_sample( file_set.keys()) for key in file_set.keys(): file = file_set[key] row_value = dict() row_key = table_name + ":" + self.make_hbasekey() byte_buffer, width, height = self.image_preprocess( file, net_info, format_info, label) row_value[':'.join( ('data', 'filebyte'))] = str(list(byte_buffer)) row_value[':'.join(('data', 'label'))] = str(label) row_value[':'.join( ('data', 'decoder'))] = str(key).split(".")[1] row_value[':'.join(('data', 'width'))] = str(width) row_value[':'.join(('data', 'height'))] = str(height) file_list.append(file._name) if (key in train_key_set): train_buffer.put(row_key, row_value) if (key in test_key_set): test_buffer.put(row_key, row_value) train_buffer.send() test_buffer.send() tfmsa_logger("[7]upload image on Hbase - finish") return file_list except Exception as e: tfmsa_logger("Error : {0}".format(e)) raise Exception(e) finally: conn.close() tfmsa_logger("Finish upload image...")
def post(self, request, baseid, tb, nnid): """ - desc : create a format data - desc : update data format information \n <textfield> <font size = 1> { "cross_cell": { "col12": {"column2_0": "native_country", "column2_1": "occupation"}, "col1": {"column_1": "occupation", "column_0": "education"} }, "cell_feature": { "hours_per_week": "CONTINUOUS_COLUMNS", "native_country": "CATEGORICAL", "relationship": "CATEGORICAL", "gender": "CATEGORICAL", "age": "CONTINUOUS_COLUMNS", "marital_status": "CATEGORICAL", "race": "CATEGORICAL", "capital_gain": "CONTINUOUS_COLUMNS", "workclass": "CATEGORICAL", "capital_loss": "CONTINUOUS_COLUMNS", "education": "CATEGORICAL", "education_num": "CONTINUOUS_COLUMNS", "occupation": "CATEGORICAL" }, "label": { "income_bracket" : "LABEL" } } </font> </textfield> --- parameters: - name: body paramType: body pytype: json """ try: jd = jc.load_obj_json("{}") jd.dir = baseid jd.table = tb jd.nn_id = nnid jd.preprocess = '1' jd.datadesc = 'Y' coll_format_json = dict() cell_format = str(request.body, 'utf-8') #if format info null print(len(request.body)) print(request.body) if (len(cell_format) == 2): print("request is null ###################") json_string = netconf.load_ori_format(nnid) coll_format_json = json.loads(json_string) cell_format = json_string else: print("request is not null ###################") coll_format_json = json.loads(cell_format) print("print cell format") print(cell_format) netconf.save_format(nnid, cell_format) print("dataformat called1###################") t_label = coll_format_json['label'] label_column = list(t_label.keys())[0] print("dataformat called2###################" + str(label_column)) # lable column_count check lable_list = data.DataMaster().get_distinct_label( baseid, tb, label_column) #hbase query lable_sorted_list = sorted(list(lable_list)) jd.datasets = lable_sorted_list #netconf.save_format(nnid, str(request.body,'utf-8')) result = netconf.update_network(jd) netconf.set_on_data(nnid) return_data = {"status": "200", "result": result} return Response(json.dumps(return_data)) except Exception as e: netconf.set_off_data(nnid) netconf.set_off_data_conf(nnid) return_data = {"status": "400", "result": str(e)} return Response(json.dumps(return_data))
def wdd_predict(self, nnid, filename=None): """ Wide & Deep Network predict :param nnid : network id in tfmsacore_nninfo :return: json list with predict value """ try: utils.tfmsa_logger( "((0.0 Wide & Deep Network Predict Start)) ## " + nnid + "##") json_string = WdnnCommonManager.get_all_info_json_by_nnid( self, nnid) database = str(json_string['dir']) table_name = str(json_string['table']) json_string_desc = netconf.load_ori_format( nnid) #Get format json string column_format_json_ob = json.loads( json_string_desc ) # Cast json object from string of column formats temp_label = column_format_json_ob[ 'label'] #Get label column from data formats label_column = list(temp_label.keys())[0] #should be change after deciding model directory model_dir = str(json_string['query']) _label_list = json_string['datasets'] label_list = eval(_label_list) #Cast a list from string utils.tfmsa_logger( "((0.0 Wide & Deep Network label List Get)) ## " + str(label_list) + "##") wdnn_model = WdnnCommonManager.wdnn_build(self, nnid, model_dir, False) if filename == None: #Doesn;t need this part now limit_no = 100 print("((2.Get Dataframe from Hbase)) ##Start## (" + database + " , " + table_name + " , " + label_column + ")") df = data.DataMaster().query_data(database, table_name, "a", limit_no, with_label=label_column) print("((2.Get Dataframe from Hbase)) ##End## (" + database + " , " + table_name + " , " + label_column + " , " + str(limit_no) + ")") else: utils.tfmsa_logger("((2.Get Dataframe from CSV)) ##Start##" + nnid + " , " + filename + ")") file_path = settings.FILE_ROOT + "/predict/" + nnid + "/" + filename utils.tfmsa_logger( "((2.Get Dataframe from CSV)) ##filePath###" + file_path + ")") df = pd.read_csv( tf.gfile.Open(file_path), # names=COLUMNS, skipinitialspace=True, engine="python") df['label'] = df[label_column] results = dict() #Make Return Dictionary predict_results = wdnn_model.predict( input_fn=lambda: WdnnCommonManager.input_fn(self, df, nnid)) df['predict_label'] = list(predict_results) utils.tfmsa_logger( "((3.1 Wide & Deep Network Predict Complete)) ## " + nnid + "##") le = LabelEncoder() le.fit(label_list) lable_decoder_func = lambda x: le.inverse_transform(x) df['predict_label'] = df['predict_label'].map( lable_decoder_func).astype( "str") #convert label code to label string. label_value = list(df['label'].unique()) label_encode = le.inverse_transform(label_value) utils.tfmsa_logger( "((3.1 Wide & Deep Network Predict)) ## Label Convert ##" + str(label_value) + "DECODE ---> " + str(label_encode)) resultList = df.values.tolist() resultList.insert(0, df.columns.values.tolist()) results = json.loads(json.dumps(resultList)) return results except Exception as e: print("Error Message : {0}".format(e)) raise Exception(e)
def run_wdd_train(self, nnid, start_pnt=1, batch_size=1000): """ Wide & Deep Network Training :param nnid : network id in tfmsacore_nninfo :return: acturacy """ try: tfmsa_logger("[2] start run wdd_Train " + nnid) #make wide & deep model wdnn_model = WdnnCommonManager.wdnn_build(self, nnid=nnid) #get json from postgres by nnid json_string = WdnnCommonManager.get_all_info_json_by_nnid( self, nnid=nnid) database = json_string["dir"] table_name = json_string["table"] #Make NetworkConfiguration Json Objct json_string = netconf.load_ori_format(nnid) json_ob = json.loads(json_string) #get label column from hbase nn config json t_label = json_ob["label"] label_column = list(t_label.keys())[0] #get train hyper param job_parm = JobStateLoader().get_selected_job_info(nnid) batch_size = int(job_parm.batchsize) model_lint_cnt = int(job_parm.epoch) tfmsa_logger( "[3] Get Dataframe from Hbase ##Start## {0},{1},{2},{3} ". format(start_pnt, database, table_name, label_column)) df, pnt = data.DataMaster().query_data(database, table_name, "a", use_df=True, limit_cnt=batch_size, with_label=label_column, start_pnt=start_pnt) df_eval = df.copy() tfmsa_logger("[4] Get Dataframe from Hbase ##End## (" + str(batch_size) + ")") ##MAKE MONITOR tfmsa_logger("[5] Make Monitor Class") customsMonitor = Monitors.MonitorCommon(p_nn_id=nnid, p_max_steps=model_lint_cnt, p_every_n_steps=1000) tfmsa_logger("[6] start fitting") wdnn_model.fit( input_fn=lambda: WdnnCommonManager.input_fn(self, df, nnid), steps=model_lint_cnt, monitors=[customsMonitor]) if (len(df_eval) < 10): tfmsa_logger("[7] Train Result") results = wdnn_model.evaluate( input_fn=lambda: WdnnCommonManager.input_fn( self, df_eval, nnid), steps=1) for key in sorted(results): tfmsa_logger("%s: %s" % (key, results[key])) return nnid else: JobStateLoader().inc_job_data_pointer(nnid) self.run_wdd_train(nnid=nnid, start_pnt=pnt) return nnid except Exception as e: print("Error Message : {0}".format(e)) raise Exception(e)