def run():
print('加载数据开始')
now = time.time()
rec_table = load_origin_data(dbug_paht)
recNum = len(rec_table)
print('原始数据：\n', rec_table)
print('总记录数：%d' % (recNum))
R = np.zeros([142, 4500, 64]) \
u = rec_table[:,0].astype(int)
s = rec_table[:, 1].astype(int)
t = rec_table[:, 2].astype(int)
R[u, s, t] = rec_table[:, 3]
print('加载数据完成，耗时 %.2f秒\n' % ((time.time() - now)))
for rid in range(epon):
allret = {}
for spa in spas:
print('开始 round-%d,spa-%.1f' % (rid, spa))
print('分割数据集开始')
now = time.time()
train, test = spliter(R, spa, rid)
print('分割数据集结束，耗时 %.2f秒\n' % ((time.time() - now)))
print('预测开始')
now = time.time()
predict = np.zeros_like(train)
for t in range(64):
print('时间片%d预测开始' % t)
predict[:, :, t] = upcc(train[:, :, t], 10)
print(mae(test[:, :, t], predict[:, :, t]))
print('预测结束，耗时 %.2f秒\n' % ((time.time() - now)))
print('评测开始')
now = time.time()
ret = evel(test, predict)
allret[spa] = ret
print('评测结束，耗时 %.2f秒\n' % ((time.time() - now)))
print(ret)
for spa in allret:
retstr = '==================================\n'
retstr += 'round-%d spa-%.2f\n' % (rid, spa)
rec = allret[spa]
retstr += 'all - mae \t%f\n' % (rec[0])
for i in range(1, len(rec)):
retstr += 't-%d mae \t%f\n' % (i, rec[i])
retstr += 'localtime \t%s\n' % (time.asctime())
retstr += '==================================\n\n'
print(retstr)
fwrite.fwrite_append(result, retstr)
pass
def run():
print ('加载数据开始');
now = time.time();
rec_table = load_origin_data(dbug_paht);
recNum = len(rec_table);
print('原始数据：\n',rec_table);
print('总记录数：%d'%(recNum))
R = np.zeros([142,4500,64]);\
u = rec_table[:,0].astype(int);
s = rec_table[:,1].astype(int);
t = rec_table[:,2].astype(int);
R[u,s,t]=rec_table[:,3];
print ('加载数据完成，耗时 %.2f秒\n'%((time.time() - now)));
for rid in range(epon):
allret = {};
for spa in spas:
print('开始 round-%d,spa-%.1f'%(rid,spa));
print('分割数据集开始');
now = time.time();
train,test = spliter(R,spa,rid);
print('分割数据集结束，耗时 %.2f秒\n'%((time.time() - now)))
print('预测开始');
now = time.time();
predict = baseline_predict(train);
print('预测结束，耗时 %.2f秒\n'%((time.time() - now)))
print('评测开始');
now = time.time();
ret = evel(test,predict);
allret[spa] = ret;
print('评测结束，耗时 %.2f秒\n'%((time.time() - now)))
print(ret);
for spa in allret:
retstr = '==================================\n';
retstr+= 'round-%d spa-%.2f\n'%(rid,spa);
rec = allret[spa];
retstr+= 'all - mae \t%f\n'%(rec[0]);
for i in range(1,len(rec)):
retstr+= 't-%d mae \t%f\n'%(i,rec[i]);
retstr+='localtime \t%s\n'%(time.asctime());
retstr+= '==================================\n\n';
print(retstr);
fwrite.fwrite_append(result, retstr);
pass;
def write2file(res):
for li in res:
fwrite.fwrite_append(loc_class_out, utils.arr2str(li))
def classf(carr, tagdir):
res = []
for idx in tagdir:
if tagdir[idx][1] in carr:
res.append(idx)
fwrite.fwrite_append(loc_class_out, utils.arr2str(res))
def train(self, NcfTraParm):
k = NcfTraParm.classif_size
class_parm = [{} for _ in range(k)]
for i in range(k):
with tf.name_scope('class%d' % (i)):
train_data = reoge_data(NcfTraParm.train_data[i])
test_data = reoge_data(NcfTraParm.test_data[i])
testn = len(test_data[0])
global_step = tf.Variable(0, trainable=False, name='gs')
class_parm[i]['global_step'] = global_step
ds = tf.data. \
Dataset.from_tensor_slices(train_data)
ds = ds.shuffle(1000).batch(NcfTraParm.batch_size)
test_ds = tf.data.Dataset.from_tensor_slices(test_data)
test_ds = test_ds.batch(testn)
it = tf.data.Iterator.from_structure(ds.output_types,
ds.output_shapes)
feat, Y = it.get_next()
train_init_op = it.make_initializer(ds)
test_init_op = it.make_initializer(test_ds)
class_parm[i]['train_init_op'] = train_init_op
class_parm[i]['test_init_op'] = test_init_op
_, loss, tmae, trmse = self.create_model(
feat, Y, self.create_param)
class_parm[i]['loss'] = loss
class_parm[i]['tmae'] = tmae
class_parm[i]['trmse'] = trmse
# loss+=tf.losses.get_regularization_loss();
lr = tf.train.exponential_decay(NcfTraParm.learn_rate,
global_step,
NcfTraParm.lr_decy_step,
NcfTraParm.lr_decy_rate,
staircase=True)
train_step = tf.train.AdagradOptimizer(lr). \
minimize(loss, global_step )
class_parm[i]['train_step'] = train_step
# summ_meg = tf.summary.merge_all();
save = tf.train.Saver()
with tf.Session() as sess:
# train_summ = tf.summary.FileWriter(NcfTraParm.summary_path+'/train',sess.graph);
# test_summ =tf.summary.FileWriter(NcfTraParm.summary_path+'/test');
if NcfTraParm.load_cache_rec:
save.restore(sess, NcfTraParm.cache_rec_path)
else:
sess.run(tf.global_variables_initializer())
now = time.time()
eptime = now
for ep in range(NcfTraParm.epoch):
test_tmae = 0.0
cot = 0
for i in range(k):
sess.run(class_parm[i]['train_init_op'])
while True:
try:
_, vloss, gs = sess.run(
(class_parm[i]['train_step'],
class_parm[i]['loss'],
class_parm[i]['global_step']))
if gs % (500) == 0:
print(
'ep%d\t class%d\t loopstep:%d\t time:%.2f\t loss:%f'
% (ep, i, gs, time.time() - now, vloss))
# summ = sess.run((summ_meg));
# train_summ.add_summary(summ, gs);
now = time.time()
except tf.errors.OutOfRangeError:
break
sess.run(class_parm[i]['test_init_op'])
# summ,vmae,vrmse,vloss=sess.run((summ_meg,tmae,trmse,loss));
vmae, vrmse, vloss = sess.run(
(class_parm[i]['tmae'], class_parm[i]['trmse'],
class_parm[i]['loss']))
test_tmae += vmae * len(NcfTraParm.test_data[i])
cot += len(NcfTraParm.test_data[i])
print(vmae)
# test_summ.add_summary(summ, ep);
vmae = test_tmae / cot
eps = '==================================================\n'
eps += 'ep%d结束 \t eptime=%.2f\n' % (ep, time.time() - eptime)
eps += 'test_mae=%f test_rmse=%f\n' % (vmae, vrmse)
eps += 'acttime=%s\n' % (time.asctime())
eps += '==================================================\n'
eptime = time.time()
print(eps)
if NcfTraParm.result_file_path != '':
fwrite_append(NcfTraParm.result_file_path, eps)
print('ep%d结束 \t eponloss=%f\t test_mae=%f test_rmse=%f\n' %
(ep, vloss, vmae, vrmse))
if NcfTraParm.cache_rec_path != '':
save.save(sess, NcfTraParm.cache_rec_path)
pass
predict[:,:,t] = pmf(train[:,:,t],32,50,0.001,0.01);
print(mae(test[:,:,t], predict[:,:,t]));
print('预测结束，耗时 %.2f秒\n'%((time.time() - now)))
print('评测开始');
now = time.time();
ret = evel(test,predict);
allret[spa] = ret;
print('评测结束，耗时 %.2f秒\n'%((time.time() - now)))
print(ret);
for spa in allret:
retstr = '==================================\n';
retstr+= 'round-%d spa-%.2f\n'%(rid,spa);
rec = allret[spa];
retstr+= 'all - mae \t%f\n'%(rec[0]);
for i in range(1,len(rec)):
retstr+= 't-%d mae \t%f\n'%(i,rec[i]);
retstr+='localtime \t%s\n'%(time.asctime());
retstr+= '==================================\n\n';
print(retstr);
fwrite.fwrite_append(result, retstr);
pass;
if __name__ == '__main__':
for rid in range(eponch):
for spa in spas:
run(spa,case,rid);
pass
def train(self,NcfTraParm3D):
# 去除无效数据，分割特征和标签
train_data = reoge_data3D(NcfTraParm3D.train_data);
test_data = reoge_data3D(NcfTraParm3D.test_data);
testn = len(test_data[0]);
print(testn);
global_step = tf.Variable(0,trainable=False,name='gs');
ds = tf.data. \
Dataset.from_tensor_slices(train_data);
ds = ds.shuffle(100).batch(NcfTraParm3D.batch_size);
test_ds = tf.data.Dataset.from_tensor_slices(test_data);
test_ds = test_ds.batch(testn//20);
it = tf.data.Iterator.from_structure(ds.output_types,
ds.output_shapes);
feat,Y = it.get_next();
train_init_op = it.make_initializer(ds);
test_init_op = it.make_initializer(test_ds);
Py,loss,tmae,trmse= self.create_model(feat, Y, self.create_param);
tf.summary.scalar('loss',loss);
tf.summary.scalar('mae',tmae);
tf.summary.scalar('rmse',trmse);
# loss+=tf.losses.get_regularization_loss;
lr = tf.train.exponential_decay(NcfTraParm3D.learn_rate, global_step,
NcfTraParm3D.lr_decy_step,
NcfTraParm3D.lr_decy_rate,
staircase=True);
train_step = tf.train.AdagradOptimizer(lr). \
minimize(loss, global_step );
summar_meg = tf.summary.merge_all();
save = tf.train.Saver();
with tf.Session() as sess:
train_summ=tf.summary.FileWriter(NcfTraParm3D.summary_path+'/train',sess.graph);
test_summ = tf.summary.FileWriter(NcfTraParm3D.summary_path+'/test',sess.graph);
if NcfTraParm3D.load_cache_rec:
save.restore(sess,NcfTraParm3D.cache_rec_path);
else:
sess.run(tf.global_variables_initializer());
now = time.time();
eptime = now;
for ep in range(NcfTraParm3D.epoch):
sess.run(train_init_op);
while True:
try:
_,vloss,gs=sess.run((train_step,loss,global_step));
if gs%(1000) == 0:
print('ep%d\t loopstep:%d\t time:%.2f\t loss:%f'%(ep,gs,time.time()-now,vloss))
now=time.time();
summ = sess.run(summar_meg)
train_summ.add_summary(summ,gs);
except tf.errors.OutOfRangeError:
break
sess.run(test_init_op);
summae=0;
while True:
try:
summ,vmae,vrmse,vloss=sess.run((summar_meg,tmae,trmse,loss));
summae+=vmae;
except tf.errors.OutOfRangeError:
break;
summae/=20;
eps = '==================================================\n'
eps += 'ep%d结束 \t eptime=%.2f\n' %(ep ,time.time()-eptime);
eps += 'test_mae=%f test_rmse=%f\n'%(summae,vrmse);
eps += 'acttime=%s\n'%(time.asctime());
eps += '==================================================\n'
eptime = time.time();
print(eps);
if NcfTraParm3D.result_file_path != '':
fwrite_append(NcfTraParm3D.result_file_path,eps);
if NcfTraParm3D.cache_rec_path != '':
save.save(sess,NcfTraParm3D.cache_rec_path)
train_summ.close();
test_summ.close();
def conbine_train(self,NcfTraParmUST):
################################ 定义 ncf ##################################
# 去除无效数据，分割特征和标签
train_data = reoge_data3D(NcfTraParmUST.train_data);
test_data = reoge_data3D(NcfTraParmUST.test_data);
testn = len(test_data[0]);
print(testn);
global_step = tf.Variable(0,trainable=False,name='gs');
ds = tf.data. \
Dataset.from_tensor_slices(train_data);
ds = ds.shuffle(1000).batch(NcfTraParmUST.batch_size);
test_ds = tf.data.Dataset.from_tensor_slices(test_data);
test_ds = test_ds.batch(testn//20);
it = tf.data.Iterator.from_structure(ds.output_types,
ds.output_shapes);
feat,Y = it.get_next();
train_init_op = it.make_initializer(ds);
test_init_op = it.make_initializer(test_ds);
Py,loss,tmae,trmse= self.create_ncf_model(feat, Y, self.create_param);
tf.summary.scalar('loss',loss);
tf.summary.scalar('mae',tmae);
tf.summary.scalar('rmse',trmse);
# loss+=tf.losses.get_regularization_loss;
lr = tf.train.exponential_decay(NcfTraParmUST.learn_rate, global_step,
NcfTraParmUST.lr_decy_step,
NcfTraParmUST.lr_decy_rate,
staircase=True);
train_step = tf.train.AdagradOptimizer(lr). \
minimize(loss, global_step );
summar_meg = tf.summary.merge_all();
##################################### 定义 rnn #####################################
seq_len = NcfTraParmUST.seq_len;
t_strat,t_end = NcfTraParmUST.time_range;
rnn_lr = NcfTraParmUST.rnn_learn_rat;
rnn_epoch = NcfTraParmUST.rnn_epoch;
R = self.get_hid_kernel('R');
nR = R[t_strat:t_end];
splitedR = tf.reshape(nR,[-1,seq_len+1,R.shape[1]]);
print(splitedR);
rnnds = tf.data.Dataset.from_tensor_slices(splitedR);
# ds.map(lambda i:(i[:-1],i[1:]));
rnnit = rnnds.make_initializable_iterator();
nextitem = rnnit.get_next();
x,y=tf.expand_dims(nextitem[:-1],0),\
tf.expand_dims(nextitem[1:],0);
x = tf.cast(x,tf.float32);
rnnpy,rnnloss,rnnmae,rnngru=self.create_rnn_model(x,y,NcfTraParmUST);
########################## 定义 rnn 预测 #########################
# 处理需要预测时间片的时间特征
splitedR = tf.reshape(R[t_end-1],[-1,1,R.shape[1]]);
pRet = rnngru(splitedR);
pRet = tf.reshape(pRet,[1,R.shape[1]]);
newR = tf.concat([R[:t_end],pRet,R[t_end+1:]],axis=0);
newP = self.get_hid_kernel('P');
newQ = self.get_hid_kernel('Q');
predict_data = reoge_data3D(NcfTraParmUST.ts_train_data);
rnn_ds = tf.data.Dataset.from_tensor_slices(predict_data);
rnn_ds = rnn_ds.batch(len(predict_data[0])//20);
pre_it = rnn_ds.make_one_shot_iterator();
perfeat,preY = pre_it.get_next();
U,S,T = self.toUSToneHot(perfeat);
newPu = tf.matmul(U,newP);
newQs = tf.matmul(S,newQ);
newRt = tf.matmul(T,newR);
ppy = self.get_ncf_nn_front(newPu, newQs, newRt,
self.create_param.hid_units,
(tf.nn.relu,tf.nn.relu), None);
print(tf.trainable_variables())
pre_mae = tf.reduce_mean(tf.abs(ppy-preY));
Ret = R[t_end];
train_op = tf.train.AdamOptimizer(rnn_lr).minimize(rnnloss);
save = tf.train.Saver();
with tf.Session() as sess:
train_summ=tf.summary.FileWriter(NcfTraParmUST.summary_path+'/train',sess.graph);
test_summ = tf.summary.FileWriter(NcfTraParmUST.summary_path+'/test',sess.graph);
if NcfTraParmUST.load_cache_rec:
save.restore(sess,NcfTraParmUST.cache_rec_path);
else:
sess.run(tf.global_variables_initializer());
now = time.time();
eptime = now;
for ep in range(NcfTraParmUST.epoch):
sess.run(train_init_op);
# 训练
while True:
try:
_,vloss,gs=sess.run((train_step,loss,global_step));
if gs%(1000) == 0:
print('ep%d\t loopstep:%d\t time:%.2f\t loss:%f'%(ep,gs,time.time()-now,vloss))
now=time.time();
summ = sess.run(summar_meg)
train_summ.add_summary(summ,gs);
except tf.errors.OutOfRangeError:
break
# 评估
sess.run(test_init_op);
summae=0;
while True:
try:
summ,vmae,vrmse,vloss=sess.run((summar_meg,tmae,trmse,loss));
summae+=vmae;
except tf.errors.OutOfRangeError:
break;
summae/=20;
eps = '==================================================\n'
eps += 'ep%d结束 \t eptime=%.2f\n' %(ep ,time.time()-eptime);
eps += 'test_mae=%f test_rmse=%f\n'%(summae,vrmse);
eps += 'acttime=%s\n'%(time.asctime());
eps += '==================================================\n'
eptime = time.time();
print(eps);
if NcfTraParmUST.result_file_path != '':
fwrite_append(NcfTraParmUST.result_file_path,eps);
if NcfTraParmUST.cache_rec_path != '':
save.save(sess,NcfTraParmUST.cache_rec_path)
train_summ.close();
test_summ.close();
############## end else ###################
########################## RNN ##########################
for ep in range(rnn_epoch):
sess.run(rnnit.initializer)
maesum=0.0;
cot=0;
while True:
try:
_,vx,vy,vpy,vloss,vmae,vpret,urrr=sess.run([train_op,x,y,rnnpy,rnnloss,rnnmae,pRet,Ret])
maesum+=vmae;
cot+=1;
# print('x:\n',vx);
# print('y:\n',vy);
# print('py:\n',vpy);
# print('loss:\n',vloss);
# print('mae:\n',vmae);
# print('pRet:\n',vpret);
# print('------------------------')
except tf.errors.OutOfRangeError:break;
maesum/=cot;
# print('pRet:\n',vpret,urrr);
print('ep%d end mae=%f'%(ep,maesum));
predict_sum_mae =0.0;
predict_cot=0;
while True:
try:
vmae=sess.run(pre_mae)
predict_sum_mae+=vmae;
predict_cot+=1;
except tf.errors.OutOfRangeError:break;
print('all predict mae',predict_sum_mae/predict_cot);
