def download_file(url, save_path, override, chunksize=65535):
downloaded = 0
if os.path.exists(save_path) and override:
logger.info(save_path + ' will be overridden.')
elif os.path.exists(save_path):
# Resume download
downloaded = os.stat(save_path).st_size
local_timestamp = os.path.getctime(save_path)
logger.info("{} already exists. Send resume request after {} bytes".format(
save_path, downloaded))
# raise FileExistsError()
old_downloaded = downloaded
headers = {}
if downloaded:
headers['Range'] = 'bytes={}-'.format(downloaded)
headers['If-Unmodified-Since'] = timestamp2time(local_timestamp)
logger.info(url + ' is saved as ' + save_path)
res = requests.get(url, headers=headers, stream=True, timeout=15)
mode = 'wb+'
content_len = int(res.headers.get('content-length'))
# Check if server supports range feature, and works as expected.
if res.status_code == 206:
# Content range is in format `bytes 327675-43968289/43968290`, check
# if it starts from where we requested.
content_range = res.headers.get('content-range')
# If file is already downloaded, it will reutrn `bytes */43968290`.
if content_range and \
int(content_range.split(' ')[-1].split('-')[0]) == downloaded:
mode = 'ab+'
elif res.status_code == 416:
# 416 means Range field not support
# TODO:需要重新下载吗
logger.warning("Range not support. Redownloading...")
urlretrieve(url, save_path, reporthook=reporthook4urlretrieve)
return
elif res.status_code == 412:
# 如果所请求的资源在指定的时间之后发生了修改，那么会返回 412 (Precondition Failed) 错误。
logger.warning("Resource Changed, should override. Redownloading...")
urlretrieve(url, save_path, reporthook=reporthook4urlretrieve)
return
file_origin_size = content_len + old_downloaded
with open(save_path, mode) as f:
for chunk in res.iter_content(chunksize):
f.write(chunk)
downloaded += len(chunk)
# TODO:如何显示下载进度
flush_print('Downloading %s %.2f%%: %d | %d' % (save_path, downloaded / file_origin_size * 100,
downloaded, file_origin_size))
# urlretrieve(url, save_path, reporthook=reporthook4urlretrieve)
print()
decompress(save_path)
def message2cloud(msg):
'''
for ai100
所有群消息都存下来，任何的群都为其建立映射 群名映射
itchat.search_chatrooms(userName='******') #建立缓存
# 分解： 把头像部分抽象为函数，把群名获取抽象为函数 把缓存考虑在内
https://github.com/youfou/wxpy
'''
message2push = {}
message2push["content"] = msg["Text"]
message2push["group_user_name"] = msg["ActualNickName"]
message2push["CreateTime"] = timestamp2time(int(msg["CreateTime"]))
push_message(message2push)
def forward_message(msg, src_group, target_groups):
'''按类型发消息'''
msg["UserImg"] = '' #成为一个开关 只有开启采用
if msg["Type"] == 'Text':
'''
#print(itchat.get_friends())
# 消息入口
username = msg["ActualUserName"] # 发言用户id 群id:FromUserName
user = itchat.search_friends(userName=username)
'''
logger.info("forward_message begin") # log for debug
logger.info(("forward message : ", msg)) #这里有所有的信息
# 多给msg一个属性 at_id
message2push = {}
message2push["group_name"] = src_group._group_name
message2push["content"] = msg["Text"]
message2push["group_user_name"] = msg["ActualNickName"]
message2push["CreateTime"] = timestamp2time(int(msg["CreateTime"]))
try:
# todo 这个那个包装为一个函数 减少副作用 传入什么 ，输出什么
# 部分用户无法获取头像 chatroomid 错误 原因在itchat username
# user_head_img = itchat.get_head_img(userName=msg["ActualUserName"],chatroomUserName=src_group._group_id,picDir="/tmp/wechat_user/{}".format(img_id)) #存储到本地
# 第一层缓存，获取头像之前，先检查本轮对话中 msg["ActualUserName"]
if USE_LEANCLOUD_FOR_IMAGE:
#from localuser import LocalUserTool, UserImgCache
user_img_cache = UserImgCache()
user_img_url = get_user_img(itchat,msg,user_img_cache,src_group._group_id)
'''
group_user_id = msg["ActualUserName"]
url_get_with_user_id = user_img.get_user_img_with_user_id(group_user_id) # 第一层缓存 , 掉线后，group_user_id变化
if url_get_with_user_id:
message2push["user_img"] = url_get_with_user_id #return
#logger.info("url_get_with_user_id: %s" ,message2push["user_img"])
else:
# 重新登录user id 变了，但是img md5还有效
#logging.debug("ActualUserName :%s, src_group _group_id: %s", msg["ActualUserName"], src_group._group_id)
img_data = itchat.get_head_img(userName=msg["ActualUserName"], chatroomUserName=src_group._group_id) # .getvalue()#前头是str #有时候错误
# logging.debug(img_data)
img_md5 = hashlib.md5(buffer(img_data)).hexdigest()
url_get_with_img_md5 = user_img.get_user_img_with_img_md5(img_md5)
if url_get_with_img_md5:
message2push["user_img"] = url_get_with_img_md5
#logger.info("url_get_with_img_md5: %s", message2push["user_img"])
else:
# 如果两级缓存都没有命中,再上传头像
url_with_uploading_img = user_img.set_user_img(group_user_id, buffer(img_data))
message2push["user_img"] = url_with_uploading_img
#logger.info("url_with_uploading_img: %s",message2push["user_img"])
'''
msg["UserImg"] = message2push["user_img"] = user_img_url #.get("user_img")
except Exception as e:
logger.error("can not get user head img")
logger.error('Failed to open file', exc_info=True)
try:
# 日志系统
if USE_LEANCLOUD_FOR_LOG:
logger.info("ready to push message to cloud")
push_message(message2push)
logger.info("ready to push message to local file")
#logger.info(message2push)
logger.info("ready to push message to local db(sqlite)")
db_store.push_message(message2push)
except Exception as e:
logger.error("log error")
logger.info(str(e))
# todo 有空优化
chat_message_recorder_logger.info(message2push) #记录所有待推送到云端的信息 时间
actual_user_name = msg["ActualNickName"]
# for at id
localuser_tool = LocalUserTool()
# feature ： at_id todo:插件化
at_id = localuser_tool.get_at_id(actual_user_name)
if not at_id:
at_id = localuser_tool.set_at_id(actual_user_name)
# 改造消息对象，使其多一个at_id
msg["at_id"] = at_id
match_at_message = re.match(
r'at *(?P<message_at_id>\d+) *(?P<message_text>.*)', msg["Text"])
if match_at_message:
# 如果是at意图的消息
groupdict = match_at_message.groupdict()
message_at_id = groupdict.get("message_at_id")
message_text = groupdict.get("message_text")
actual_user_name = localuser_tool.get_actual_user_name(
int(message_at_id))
for group in target_groups:
now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
logger.info((now, group._group_name, msg[
'ActualNickName'], msg["Text"]))
# 加入群名 group._group_name 做一个映射
message = u'@{} \n{}-at_id:{} 发言 ：\n{}'.format(
actual_user_name, msg['ActualNickName'], msg['at_id'], message_text)
#message = u'@{}\u2005\n : {}'.format(actual_user_name,message_text)
itchat.send(message, group._group_id)
else:
# 如果不是at消息：即普通文本消息
# bot回复论坛消息
# handle_text_msg对消息做个分类，只做分析，handle_text_msg不主动发送消息，此处可嵌入插件
response = handle_text_msg(msg).get("response")
if response:
# 只回复本群
itchat.send(response, src_group._group_id)
else:
# 推送到其他群
for group in target_groups:
now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
logger.info((now, group._group_name, msg[
'ActualNickName'], msg["Text"]))
# if group._group_id:
message = '{}-at_id:{} 发言 ：\n{}'.format(
msg['ActualNickName'], msg['at_id'], msg['Text'])
itchat.send(message, group._group_id) # 采用异步
# 之后的消息统一转发
if msg["Type"] == 'Picture':
# todo：上传到云端
msg['Text'](msg['FileName']) # 下载 其他大文件 异步使用另一个线程
for group in target_groups:
itchat.send_image(msg['FileName'], group._group_id)
if msg['Type'] == 'Sharing':
# todo: 同样作为普通消息存入log
share_message = "@{}分享\n{} {}".format(
msg['ActualNickName'], msg["Url"].replace("amp;", ""), msg["Text"])
for group in target_groups:
itchat.send_msg(share_message, group._group_id)
def simple_reply(msg):
global groups
global other_group_map #头像也用这个思路，groupUserName不需要永久化
print("group message input from wechat(begin)")
# 互相转发的群
for group in groups:
#logger.info(("local_group", group._group_id, group._group_name))
#logger.info("msg from group:{}".format(msg['FromUserName']))
if msg['FromUserName'] == group._group_id:
# 一条消息只能匹配一次
src_group = group # 消息来源群组
target_groups = get_target_groups(src_group, tuple(groups)) # 消息发往的目标群
# 筛选出已激活的
active_target_groups = [group for group in target_groups if group._group_id]
forward_message(msg, src_group, active_target_groups) # !!内部逻辑入口
if not group._group_id:
# 如果群未被激活，则开始搜索群id
# 每条消息都找一下，维护一个全局群id列表
group_instance = itchat.search_chatrooms(name=group._group_name)
if group_instance:
group.set_id(group_instance[0]['UserName'])
print("{}激活,group_id:{}".format(group._group_name, group._group_id))
if DEBUG:
itchat.send_msg('机器人已激活: )', group._group_id)
# todo 都需要头像 拆分出来
# topic groups 只定期发送到大群
# forward_message(msg, src_group, active_target_groups)
# 也是只存储，只是定期看一下数据，然后转让，定时任务
# 定时任务 不影响主线程 https://github.com/mrhwick/schedule/blob/master/schedule/__init__.py 仅仅把schedule.run_pending()改为run_continuously()
# 使用本地数据库查询 时间用timestamp arrow
# db_store.push_message(message2push)
# 然后写查询 每2小时查询一下 pillow生成图片 使用jupyter来做
# 大体上的布局 [xx]:xxx
# 需求 摘要+链接(网页)
##############################################
# log other groups 记录群消息
# 作为群的一种类型 如果是这种类型的消息则不转发 只存储
# 获取群名，获取头像 独立函数 缓存
is_other_groups_msg = msg['FromUserName'] not in [group._group_id for group in groups]
if setting.PUSH_ALL_GROUP_MESSAGE_TO_LEANCLOUD and is_other_groups_msg:
# 没有头像
from leancloud_store import push_message
other_group_FromUserName = msg['FromUserName']
other_group_NickName = other_group_map.get(other_group_FromUserName)
# 建立缓存 维护一个字典即可，不需要永久化 todo 头像也这样做
if not other_group_NickName:
other_group = itchat.search_chatrooms(userName=other_group_FromUserName)
other_group_NickName = other_group["NickName"]
logger.debug("store other_group_NickName to other_group_map")
other_group_map[other_group_FromUserName] = other_group_NickName
message2push={}
message2push["content"] = msg["Text"]
message2push["group_user_name"] = msg["ActualNickName"]
message2push["CreateTime"] = timestamp2time(int(msg["CreateTime"]))
message2push["group_name"] = other_group_NickName
user_img_cache = UserImgCache()
user_img_url = get_user_img(itchat,msg,user_img_cache,other_group_FromUserName)
message2push["user_img"] = user_img_url
logger.info("ready to push other group message to cloud")
push_message(message2push)