def get_messages(uid, token):
#TODO: 现在只是获取第一页私信列表，并且是每个对话的最近一条(而且没判断是发送的还是接受的)。要改成获取每页私信（每个对话最近条即可）。
http = httphelper.HttpHelper()
# set cookies and gsid
http.add_cookie('_WEIBO_UID', uid, '.weibo.cn')
http.add_cookie('_T_WL', '1', '.weibo.cn')
http.add_cookie('gsid_CTandWM', token, '.weibo.cn')
try:
#get all msgs
html = http.get(config.URL_MSG)
except:
e = 'failed to GET msg', util.str_time()
logging.info(e)
try:
#parse statuses
statuses = []
idx = 0
pa_status = re.compile(r'说:(.*?) <span class="ct">')
pa_touid = re.compile(r'/im/callchat\?uid=(\d+)')
for status in re.findall(pa_status, html):
statuses.append(model.Status(content = status))
# statuses.append(status)
for touid in re.findall(pa_touid, html):
statuses[idx].uid = str(uid) #TODO 一下int64，一下string的...我已经搞不清楚了...
statuses[idx].touid = touid
idx += 1
except:
e = 'failed to PARSE statuses', util.str_time()
logging.info(e)
return statuses
def handle_status(uid, token, status):
"""处理微博并发送，处理多条发送"""
http = httphelper.HttpHelper()
# set cookies and gsid
http.add_cookie('_WEIBO_UID', uid, '.weibo.cn')
http.add_cookie('_T_WL', '1', '.weibo.cn')
http.add_cookie('gsid_CTandWM', token, '.weibo.cn')
# logging.debug(http.cj)
#make url_post
try:
html_main = http.get(config.URL_WEIBO)
pa_st = re.compile(r'(.*?)st=(.*?)&') # 不知道为什么这里会匹配到st后面的信息..
m = pa_st.match(html_main)
st = m.group(2) #/mblog/sendmblog?st=1ad7&st=1ad7&vt=4&gsid=3_58ac1a8401cf4825abacad4c983f2c5080b1e8
st = st[0:4]
url_post = '%s?%s' % (config.URL_POST, httphelper._encode_params(gsid = token,
st = st,
vt = 4))
except:
e = 'failed to MAKE url_post', util.str_time()
logging.info(e)
maxlen = config.STATUS_MAXLEN * 3
if len(status) > maxlen: #分条发送
length = maxlen-3*5
n = len(status)/length + 1
for j in range(0, n):
_post_status(http, url_post, '['+str(j+1)+'/'+str(n)+']' + status[j*length:j*length+length])
else:
_post_status(http, url_post, status[0:420]) #3倍
def message_chat(uid, token, touid):
# ?uid=1659177872
http = httphelper.HttpHelper()
# set cookies and gsid
http.add_cookie('_WEIBO_UID', uid, '.weibo.cn')
http.add_cookie('_T_WL', '1', '.weibo.cn')
http.add_cookie('gsid_CTandWM', token, '.weibo.cn')
html = None
"""
与逆风soso的聊天记录 清空
我:试试8分钟前 转发
我:testlo12分钟前 转发
我:发送测试啦50分钟前 转发
我:给逆风啦10月16日 23:26 转发
我:test10月15日 16:18 转发
逆风soso:测试下啦啦啦啦03月31日 22:21 转发
"""
try:
#get chat msgs
html = http.get(config.URL_MSG_Chat + '?type=record&uid=' + touid)
except:
e = 'get chat msgs', util.str_time()
logging.info(e)
msgs = []
try:
#parse msgs
idx = 0
pa_msg = re.compile(r'<div class="c">我:(.*?)<span class="ct">|</a>:(.*?)<span class="ct">')
# pa_msg_send = re.compile(r'<div class="c">我:(.*?)<span class="ct">')
# pa_msg_receive = re.compile(r'</a>:(.*?)<span class="ct">')
pa_msg_time = re.compile(r'<span class="ct">(.*?)</span> ')
for msg in re.findall(pa_msg, html):
type = 0 if msg[0] else 1
msgs.append(model.Message(uid = str(uid), touid = str(touid), content = msg[type], type=type))
for time in re.findall(pa_msg_time, html):
msgs[idx].time = time
idx += 1
except:
e = 'failed to parse msgs', util.str_time()
logging.info(e)
return msgs #TODO，正则获得私信列表
def train():
t1 = time.time()
tf.reset_default_graph()
with tf.variable_scope(name_or_scope='train', reuse=tf.AUTO_REUSE):
cls, (x, y), (w0, w1, w2, w3, w4) = gm.result()
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y,
logits=cls,
name='loss')
loss_mean = tf.reduce_mean(loss, name='loss_mean')
global_step = tf.Variable(0, name='global_step')
learning_rate = tf.train.exponential_decay(constant.LEARNING_RATE,
global_step,
1000,
0.96,
staircase=True,
name='learning_rate')
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate,
name='optimizer')
train_op = optimizer.minimize(loss_mean,
global_step=global_step,
name='train_op')
data_train = util.load_data(constant.DATA_TRAIN)
data_test = util.load_data(constant.DATA_TEST)
graph = tf.get_default_graph()
# var_list = [i for i in tf.global_variables() if i.name.split('/')[1] == 'result']
# saver = tf.train.Saver(var_list=var_list, max_to_keep=5)
# [print(i) for i in tf.global_variables()]
# [print(i.name) for i in graph.get_operations()]
saver = tf.train.Saver(max_to_keep=5)
with tf.Session(graph=graph) as sess:
sess.run(tf.global_variables_initializer())
idx_train = np.linspace(0,
constant.TRAIN_TOTAL_SIZE - 1,
constant.TRAIN_TOTAL_SIZE,
dtype=np.int32)
step = 0
accuracies_train = []
accuracies_test = []
losses = []
ws = (w0, w1, w2, w3, w4)
wa = WeightAdjust()
wa.init(len(ws))
for i in range(constant.EPOCH):
np.random.shuffle(idx_train)
for j in range(constant.TRAIN_TIMES_FOR_EPOCH):
idx_j = np.linspace(j * constant.BATCH_SIZE,
(j + 1) * constant.BATCH_SIZE - 1,
constant.BATCH_SIZE,
dtype=np.int32)
idx_train_batch = idx_train[idx_j]
_, labels_train, _, images_train = util.get_batch(
data_train, idx_train_batch)
feed_dict_train = {x: images_train, y: labels_train}
cls_train, _loss, _ = sess.run([cls, loss_mean, train_op],
feed_dict=feed_dict_train)
arg_idx_train = np.argmax(cls_train, axis=1)
accuracy_train = sum(
labels_train == arg_idx_train) / constant.BATCH_SIZE
# test
idx_test_batch = np.random.randint(0, constant.TEST_TOTAL_SIZE,
[constant.BATCH_SIZE])
_, labels_test, _, images_test = util.get_batch(
data_test, idx_test_batch)
feed_dict_test = {x: images_test, y: labels_test}
cls_test = sess.run(cls, feed_dict=feed_dict_test)
arg_idx_test = np.argmax(cls_test, axis=1)
accuracy_test = sum(
labels_test == arg_idx_test) / constant.BATCH_SIZE
step += 1
if step % constant.PRINT_EVERY_TIMES == 0:
print(
'time:{},epoch:{},loss:{},accuracy_train:{:.2%},accuracy_test:{:.2%}'
.format(util.cur_time(), step, _loss, accuracy_train,
accuracy_test))
accuracies_train.append(accuracy_train)
accuracies_test.append(accuracy_test)
losses.append(_loss)
times = int(constant.TRAIN_TIMES_FOR_EPOCH /
constant.PRINT_EVERY_TIMES)
train_mean = util.mean(accuracies_train[-times:])
test_mean = util.mean(accuracies_test[-times:])
print('save model,step: {},train_mean:{},test_mean:{}'.format(
step, train_mean, test_mean))
saver.save(sess,
save_path='./model/resnet/cifar-resnet.ckpt',
global_step=step)
wa.adjust(train_mean, test_mean, step)
print(wa.action)
if wa.action == 'adjust':
print('本次迭代权重经过调整：{}'.format(wa.weights))
assigns = gm.assign_weight(wa, ws)
sess.run(assigns)
elif wa.action == 'stop':
break
else:
pass
accuracy_map = {
'accuracies_train': accuracies_train,
'accuracies_test': accuracies_test,
'losses': losses,
'weights': wa
}
util.dump_data(accuracy_map, './accuracy_map.pkl')
t2 = time.time()
print('耗时：{}'.format(util.str_time(t2 - t1)))
