def run(self, agent, bts_policy):
log_handler.log.info("Reload Esmm ctcvr model")
self.model = get_trained_model()
log_handler.log.info("Reset all ctcvr score")
handler = LoadDb()
handler.reset_sku_keyword_score()
log_handler.log.info("Load sku feature and sku-keyword pair")
df_test_sample = pd.read_csv(get_data_path() + FEATURE_TEST_SOURCE,
dtype={
"pagecat": int,
"brand": int,
"cat": int,
"cat1": int
})
log_handler.log.info("Begin scoring chunks of sku-keyword pairs")
for df_chunk in list2chunks(df_test_sample, 500000):
try:
self.model.predict(df_chunk, FINAL_RESULT)
log_handler.log.info("Loading score to DB")
handler = LoadDb()
handler.read_final_result(agent, bts_policy)
handler.load_sku_keyword_score()
except Exception as e:
log_handler.log.info("Error scoring sample chunk")
log_handler.log.info(str(excp_trace()))
self.model.clear_all()
def build_raw_sample():
try:
log_handler.log.info(
"----------------Downloading raw samples for training----------------"
)
HdfsTrainLog.clear_all_data_info()
HdfsTrainLog.download_traning_from_hdfs()
HdfsTrainLog.convert_hdfs_to_list()
log_handler.log.info(
"----------------Finish downloading raw samples----------------")
time.sleep(5)
log_handler.log.info(
"----------------Downloading raw samples for testing----------------"
)
HdfsTestLog.download_testing_from_hdfs()
HdfsTestLog.convert_hdfs_to_list()
log_handler.log.info(
"----------------Finish downloading raw samples----------------")
time.sleep(5)
except Exception:
log_handler.log.info(
"----------------Error building raw samples----------------")
log_handler.log.info(str(excp_trace()))
raise Exception
def gbmain(agent, bts_policy, domain, channel, algorithm_id, redis_msg):
if GB_CTCVR_LOCK.acquire(False):
try:
log_handler.log.info("**Server**: GB CTCVR running request accepted, task lock aquired")
if check_algorithm(algorithm_id):
log_handler.log.info("**Server**: Inform DB that GB CTCVR is running")
update_algorithm(algorithm_id, 1, domain)
build_raw_sample()
download_goods_detail()
build_sample()
model_train()
load_result_main(agent, bts_policy)
log_handler.log.info("**Server**: Inform DB that GB CTCVR has finished running")
update_algorithm(algorithm_id, 2, domain)
log_handler.log.info("**Server**: Inform Redis that GB CTCVR has finished running, msg:%s" % redis_msg)
inform_redis_msg(channel, redis_msg)
log_handler.log.info("**Server**: GB CTCVR complete, mission successful")
else:
log_handler.log.info("**Server**: Algorithm with id:%s not found" % algorithm_id)
except:
error_msg = str(excp_trace())
log_handler.log.info("**Server**: Inform DB that GB CTCVR running fatal：%s" % error_msg)
update_algorithm(algorithm_id, 3, domain)
finally:
GB_CTCVR_LOCK.release()
log_handler.log.info("**Server**: Request done processing, task lock released")
else:
log_handler.log.info("**Server**: GB CTCVR running request rejected, for another instance is already running!!!")
def download_goods_detail():
try:
log_handler.log.info("----------------Building goods test info----------------")
SkuEsData.clear_all_goods_info()
SkuEsData.build_all_goods_info()
SkuEsData.filter_all_goods_info()
log_handler.log.info("----------------Finish downloading search test log----------------")
except Exception:
log_handler.log.info("----------------Error building goods test info----------------")
log_handler.log.info(str(excp_trace()))
raise Exception
def load_result_main(agent, bts_policy):
try:
log_handler.log.info(
"----------------Processing batch result----------------")
task_handler = LoadResultMain()
task_handler.run(agent, bts_policy)
log_handler.log.info(
"----------------Finish processing batch result----------------")
time.sleep(5)
except Exception:
log_handler.log.info(
"----------------Error building batch result----------------")
log_handler.log.info(str(excp_trace()))
raise Exception
def update_algorithm(algorithm_id, status, domain):
try:
log_handler.log.info(
"----------------Update algorithm id:%s with status %s----------------"
% (algorithm_id, status))
handler = LoadDb()
log_handler.log.info(
"----------------Done updating algorithm status----------------")
return handler.update_algorithm_status(algorithm_id, status, domain)
except Exception:
log_handler.log.info(
"----------------Error updating algorithm status----------------")
log_handler.log.info(str(excp_trace()))
raise Exception
def load_to_db(agent, bts_policy):
try:
log_handler.log.info(
"----------------Loading search sku score----------------")
handler = LoadDb()
handler.read_final_result(agent, bts_policy)
handler.load_sku_keyword_score()
log_handler.log.info(
"----------------Done loading search sku score----------------")
except Exception:
log_handler.log.info(
"----------------Error loading search sku score----------------")
log_handler.log.info(str(excp_trace()))
raise Exception
def model_train():
try:
log_handler.log.info(
"----------------Training Esmm Model----------------")
rmdir_if_exist(get_data_path() + "../saved_sessions/result/model")
esmm_model_train(FEATURE_TRAIN_SOURCE)
log_handler.log.info(
"----------------Finish training Esmm Model----------------")
time.sleep(5)
except Exception:
log_handler.log.info(
"----------------Error training Esmm Model----------------")
log_handler.log.info(str(excp_trace()))
raise Exception
def check_algorithm(algorithm_id):
try:
log_handler.log.info(
"----------------Check if algorithm with id:%s exist----------------"
% algorithm_id)
handler = LoadDb()
log_handler.log.info(
"----------------Done checking algorithm existence----------------"
)
return handler.algorithm_exist(algorithm_id)
except Exception:
log_handler.log.info(
"----------------Error checking algorithm existence----------------"
)
log_handler.log.info(str(excp_trace()))
raise Exception
def build_goods_info():
try:
log_handler.log.info(
"----------------Building goods info----------------")
features = [
"goodsSn", "brandName", "categories", "discount", "shopPrice",
"displayPrice", "stockFlag", "youtube", "totalFavoriteCount",
"passTotalNum", "passAvgScore", "exposureSalesRate", "grossMargin",
"exposureSalesVolume", "week2Sales", "week2SalesVolume",
"dailyRate", "yesterdaySales", "goodsWebSpu"
]
data_handler = GoodssnAllHandler(features)
data_handler.build_goods()
time.sleep(10) # 睡眠10秒以等待商品列表已成功写入磁盘
except Exception:
log_handler.log.info(
"----------------Error building goods info----------------")
log_handler.log.info(str(excp_trace()))
raise Exception
def build_sample():
try:
log_handler.log.info(
"----------------Building id dictionaries----------------")
BuildSample.build_id_dict()
log_handler.log.info(
"----------------Finish building id dictionaries----------------")
time.sleep(5)
log_handler.log.info(
"----------------Building all goods features----------------")
BuildSample.make_good_feature()
log_handler.log.info(
"----------------Finish building all goods features----------------"
)
time.sleep(5)
log_handler.log.info(
"----------------Building training sample----------------")
BuildSample.make_training_sample()
log_handler.log.info(
"----------------Finish building all training sample----------------"
)
time.sleep(5)
log_handler.log.info(
"----------------Building testing sample----------------")
BuildSample.make_testing_sample()
log_handler.log.info(
"----------------Finish building testing sample----------------")
time.sleep(5)
except Exception:
log_handler.log.info(
"----------------Error building sample----------------")
log_handler.log.info(str(excp_trace()))
raise Exception
