if name != "grid_size":
tensorboard_log += [("{}_{}".format(name,j+1), metric)]
tensorboard_log += [("loss", loss.item())]
logger.list_of_scalars_summary(tensorboard_log, batches_done)
# log_str += AsciiTable(metric_table).table
# print(metric_table)
log_str +='\n'.join(['\t'.join(line) for line in metric_table])
log_str += "\nTotal loss {}".format(loss.item())
# Determine approximate time left for epoch
epoch_batches_left = len(dataloader) - (batch_i + 1)
time_left = datetime.timedelta(seconds=epoch_batches_left * (time.time() - start_time) / (batch_i + 1))
log_str += "\n---- ETA {}".format(time_left)
local_logger.info(log_str)
model.seen += imgs.size(0)
if epoch % opt.evaluation_interval == 0:
local_logger.info("\n---- Evaluating Model ----")
# Evaluate the model on the validation set
precision, recall, AP, f1, ap_class = evaluate(
model,
path=valid_path,
iou_thres=0.5,
conf_thres=0.5,
nms_thres=0.5,
img_size=opt.img_size,
batch_size=8,
mydataset=img_label_map
class Weibo(object):
def __init__(self, account_file):
self.check_paths(['log', 'pic', 'cookies', 'driver', 'account'])
self.username, self.password = self.get_account(account_file)
self.login_api = 'https://passport.weibo.cn/sso/login'
self.user_info_api = 'https://m.weibo.cn/home/me?format=cards'
self.user_weibo_api = "https://m.weibo.cn/api/container/getIndex?uid={}&luicode=20000174&type=uid&value={}&containerid=107603{}"
self.headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:56.0) Gecko/20100101 Firefox/56.0',
}
self.logger = MyLogger('weibo').get_logger()
self.cookie_path = 'cookies'
self.conn = redis.Redis(connection_pool=redis.ConnectionPool(host='redis', port=6379, decode_responses=True))
self.cookies = self.get_cookies()
def get_account(self, filename):
with open(filename, 'r') as f:
d = json.load(f)
return d.get('username'), d.get('password')
def check_paths(self, paths):
for p in paths:
if not os.path.exists(p):
os.mkdir(p)
def make_headers(self, items=None):
headers = self.headers.copy()
if items:
headers.update(items)
return headers
def login(self):
login_data = {
'username': self.username,
'password': self.password,
'savestate': 1,
'r': 'http%3A%2F%2Fm.weibo.cn%2F',
'ec': 0,
'pagerefer': '',
'entry': 'mweibo',
'wentry': '',
'loginfrom': '',
'client_id': '',
'code': '',
'qq': '',
'mainpageflag': 1,
'hff': '',
'hfp': ''
}
items = {
'Referer': 'https://passport.weibo.cn/signin/login?entry='
'mweibo&res=wel&wm=3349&r=https%3A%2F%2Fm.weibo.cn%2F'
}
try:
r = requests.post(self.login_api, headers=self.make_headers(items), data=login_data)
if r.json()['retcode'] == 20000000:
self.logger.info('登录成功')
r.cookies.pop('login')
self.save_cookie(r.cookies)
return r.cookies
else:
self.logger.error('登录失败, 可能需要验证码 {}'.format(r.text))
self.logger.info('正在尝试用selenium登录')
cookies = selenium_login(self.username, self.password)
if not cookies:
self.logger.error('selenium登录失败')
exit(1)
self.logger.info('selenium登录成功')
self.save_cookie(cookies)
return cookies
except Exception as e:
self.logger.error('登录失败 {}'.format(e), exc_info=True)
def save_cookie(self, cookies):
file = os.path.join(self.cookie_path, self.username + '.pkl')
try:
with open(file, 'wb') as f:
self.logger.info('cookie保存成功')
pickle.dump(cookies, f)
except Exception as e:
self.logger.error('保存cookie失败 {}'.format(e), exc_info=True)
def get_cookies(self):
file = os.path.join(self.cookie_path, self.username + '.pkl')
try:
with open(file, 'rb') as f:
cookies = pickle.load(f)
if self.verify_cookie(cookies):
self.logger.info('cookie验证成功')
return cookies
else:
self.logger.warning('cookie过期')
self.logger.warning('正在重新登录')
cookies = self.login()
if not cookies:
self.logger.error('登陆失败', exc_info=True)
except Exception as e:
self.logger.error('cookie不存在，重新登录 {}'.format(e), exc_info=True)
cookies = self.login()
if not cookies:
exit(1)
return cookies
def verify_cookie(self, cookies):
try:
r = requests.get(self.user_info_api, headers=self.make_headers(), cookies=cookies)
data = r.json()
user_info = data[0]["card_group"][0]["user"]
if user_info:
return True
except Exception as e:
self.logger.error('验证cookie失败 {}'.format(e), exc_info=True)
return False
def get_user_basic_info(self):
try:
r = requests.get(self.user_info_api, headers=self.make_headers(), cookies=self.cookies)
data = r.json()
user_info = data[0]["card_group"][0]["user"]
user = dict()
user["user_id"] = user_info["id"]
user["user_name"] = user_info["name"]
user["weibo_count"] = user_info["mblogNum"]
user["follow_count"] = user_info["attNum"]
return user
except Exception as e:
self.logger.error('获取个人信息失败 {}'.format(e), exc_info=True)
return
def get_user_weibo(self, uid): # 获取前十条微博
self.logger.info('正在获取 {}的微博'.format(uid))
url = self.user_weibo_api.format(uid, uid, uid)
try:
r = requests.get(url, headers=self.make_headers(), cookies=self.cookies)
cards = r.json().get("data").get("cards")
for card in cards:
# weibo = {}
try:
# weibo["weibo_content_id"] = card.get("mblog").get("id")
# weibo["weibo_content"] = card.get("mblog").get("text")
# weibo["weibo_user_id"] = card.get("mblog").get("user").get("id")
# weibo["weibo_username"] = card.get("mblog").get("user").get("screen_name")
# weibo["mid"] = card.get("mblog").get("mid")
wid = card.get("mblog").get("id")
mid = card.get("mblog").get("mid")
self.logger.info('{} - {}'.format(wid, mid))
yield wid, mid
except AttributeError:
continue # cards列表里面不一定是微博，用try来过滤
except Exception as e:
self.logger.error('获取用户微博失败 {}'.format(e))
def get_st(self): # st是转发微博post必须的参数
url = "https://m.weibo.cn/api/config"
# self.s.get(url, headers=self.make_cookie_header(), cookies=self.cookies, proxies=self.proxies, verify=False)
try:
s = requests.session()
r = s.get(url, headers=self.make_headers(), cookies=self.cookies)
data = r.json()
st = data["data"]["st"]
return st, s
except Exception as e:
self.logger.error('http error {}'.format(e), exc_info=True)
def forward_weibo(self, weibo, content):
st, s = self.get_st()
url = "https://m.weibo.cn/api/statuses/repost"
data = {"id": weibo["wid"], "content": content, "mid": weibo["mid"], "st": st}
# 这里一定要加referer， 不加会变成不合法的请求
items = {
'Referer': 'https://m.weibo.cn/compose/repost?id={}'.format(weibo['wid'])
}
try:
r = s.post(url, data=data, headers=self.make_headers(items), cookies=self.cookies)
if r.json().get("ok") == 1:
self.logger.info('转发微博{}成功'.format(weibo['wid']))
return True
else:
self.logger.warning('转发微博{}失败 {}'.format(weibo['wid'], r.text))
return None
except Exception as e:
self.logger.error('http error {}'.format(e), exc_info=True)
return None
# 发送原创微博
def send_original_weibo(self, content, pic_path=None):
st, s = self.get_st()
data = {
# "luicode": "10000011",
# "lfid": "2304135827525376_ - _WEIBO_SECOND_PROFILE_MORE_WEIBO",
# "featurecode": "20000320",
"content": content,
"st": st
}
if pic_path:
pic_id = self.upload_pic(pic_path)
if pic_id:
data["picId"] = pic_id
url = "https://m.weibo.cn/api/statuses/update"
items = {
'Referer': 'https://m.weibo.cn/compose/'
}
try:
r = s.post(url, data=data, headers=self.make_headers(items), cookies=self.cookies)
if r.json()["ok"] == 1:
self.logger.info('原创微博发送成功')
else:
self.logger.warning('原创微博发送失败')
except Exception as e:
self.logger.error('http error {}'.format(e))
return None
def upload_pic(self, pic_path):
url = "https://m.weibo.cn/api/statuses/uploadPic"
st, s = self.get_st()
pic_name = os.path.split(pic_path)[-1]
# 这里如果pic_name 中有 '\' 会上传失败， 在windows里 \是路径，要去掉
try:
files = {"pic": (pic_name, open(pic_path, "rb").read(), "image/jpeg")}
except Exception as e:
self.logger.error('打开图片失败 {}'.format(e), exc_info=True)
return None
items = {
'Referer': 'https://m.weibo.cn/compose/'
}
data = {"type": "json", "st": st}
# print(r.json())
try:
r = s.post(url, data=data, files=files, headers=self.make_headers(items), cookies=self.cookies)
pic_id = r.json()["pic_id"]
self.logger.info('图片上传成功')
return pic_id
except Exception as e:
self.logger.error('图片上传失败 {}'.format(e), exc_info=True)
return None
logCount = 0
maxLogCount = int(30 / delay)
t = temp.read()
currentStatus = fan.read()
def strStatus(status=None):
if status == None:
status = currentStatus
return "On" if status else "Off"
logger.info(
"STARTING: Fan=%3s, Delay=%.1f, Threshold=%.1f/%.1f, MaxCount=%d/%d, MaxLogCount=%d"
% (strStatus(), delay, thresholdUp, thresholdDown, maxCountUp,
maxCountDown, maxLogCount))
while t:
newStatus = t > (thresholdDown if currentStatus else thresholdUp)
if newStatus != currentStatus:
count += 1
else:
count = 0
if (logCount % maxLogCount == 0) or (count > 0):
logCount = 0
logger.debug("Fan=%3s, Count=%d, Temp=%2.3f", strStatus(), count, t)
if count >= (maxCountDown if currentStatus else maxCountUp):
logCount = 0
maxLogCount = int(30 / delay)
t = temp.read()
currentStatus = fan.read()
def strStatus(status = None):
if status == None:
status = currentStatus
return "On" if status else "Off"
logger.info("STARTING: Fan=%3s, Delay=%.1f, Threshold=%.1f/%.1f, MaxCount=%d/%d, MaxLogCount=%d" % (
strStatus(), delay,
thresholdUp, thresholdDown,
maxCountUp, maxCountDown,
maxLogCount))
while t:
newStatus = t > (thresholdDown if currentStatus else thresholdUp)
if newStatus != currentStatus:
count += 1
else:
count = 0
if (logCount % maxLogCount == 0) or (count > 0):
logCount = 0
logger.debug("Fan=%3s, Count=%d, Temp=%2.3f", strStatus(), count, t)