def recover(self):
"""
@brief Recover from snapshot data.
@return Bool: whether operation succeed.
"""
cm = ConfigManager()
[self.uuid, self.redirect_uri, self.uin,
self.sid, self.skey, self.pass_ticket,
self.synckey, device_id, self.last_login] = cm.get_wechat_config()
if device_id:
self.device_id = device_id
self.base_request = {
'Uin': int(self.uin),
'Sid': self.sid,
'Skey': self.skey,
'DeviceID': self.device_id,
}
# set cookie
Log.debug('set cookie')
self.cookie = set_cookie(self.cookie_file)
return True
def get(url, api=None):
"""
@brief http get request
@param url String
@param api wechat api
@return http response
"""
Log.debug('GET -> ' + url)
request = urllib2.Request(url=url)
request.add_header(*Constant.HTTP_HEADER_CONNECTION)
request.add_header(*Constant.HTTP_HEADER_REFERER)
if api in ['webwxgetvoice', 'webwxgetvideo']:
request.add_header(*Constant.HTTP_HEADER_RANGE)
while True:
try:
response = urllib2.urlopen(request, timeout=30)
data = response.read()
response.close()
if api == None:
Log.debug(data)
return data
except (KeyboardInterrupt, SystemExit):
raise
except:
Log.error(traceback.format_exc())
time.sleep(1)
def delete_table(self, table):
"""
@brief Delete a table in database
@param table String
"""
sql = "DROP TABLE if exists %s;" % table
Log.debug('DB -> %s' % sql)
self.execute(sql)
def delete_table(self, table):
"""
@brief Delete a table in database
@param table String
"""
sql = "DROP TABLE if exists %s;" % table
Log.debug('DB -> %s' % sql)
self.execute(sql)
def doActionAgentCollate(self, action_sequence):
# Given an action space to perform, need to convert it into the
# proper representation that the map can understand. In this case,
# there are PUT and DEL actions.
# Retrieve the current position of the agent within the active map
current_pos = self.active_map.getAgentMapInfo(self)
map_actions = defaultdict(list)
for action in action_sequence:
if (action == Agent.AgentActions.NO_MOVE):
# The agent has decided to make no map-oriented moves
pass
elif (action == Agent.AgentActions.TURN_UP):
# The agent is modifying an internal state and has no map updates
# that it needs to report back to the map
self.orientation = Agent.Orientations.UP
elif (action == Agent.AgentActions.TURN_DOWN):
# The agent is modifying an internal state and has no map updates
# that it needs to report back to the map
self.orientation = Agent.Orientations.DOWN
elif (action == Agent.AgentActions.TURN_LEFT):
# The agent is modifying an internal state and has no map updates
# that it needs to report back to the map
self.orientation = Agent.Orientations.LEFT
elif (action == Agent.AgentActions.TURN_RIGHT):
# The agent is modifying an internal state and has no map updates
# that it needs to report back to the map
self.orientation = Agent.Orientations.RIGHT
elif (action == Agent.AgentActions.FORWARD):
# Add the movement based on the current orientation of the
# agent.
new_x, new_y = current_pos.x, current_pos.y
if (self.orientation == Agent.Orientations.UP):
new_y -= 1
elif (self.orientation == Agent.Orientations.DOWN):
new_y += 1
elif (self.orientation == Agent.Orientations.LEFT):
new_x -= 1
elif (self.orientation == Agent.Orientations.RIGHT):
new_x += 1
else:
raise AssertionError("Invalid Orientation")
# If the new position is valid, move
new_pos = self.active_map.GridPosition(new_x, new_y)
if self.active_map.isValidPosition(new_pos):
map_actions[new_pos].append(
self.active_map.PutMapUpdate(self))
# Delete the agent from the current location
map_actions[current_pos].append(
self.active_map.DelMapUpdate(self))
else:
Log.debug(
"invalid new map position - not generating anything")
else:
raise AssertionError("An invalid action has been specified")
return map_actions
def insert(self, table, value):
"""
@brief Insert a row in table
@param table String
@param value Tuple
"""
sql = ("INSERT INTO %s VALUES (" + ",".join(['?'] * len(value)) + ");") % table
Log.debug('DB -> %s' % sql)
self.execute(sql, value)
def create_db(self, db_name):
"""
@brief Creates a database
@param db_name String
"""
if self.conf['database'] not in self.show_database():
sql = 'CREATE DATABASE IF NOT EXISTS %s CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci' % db_name
Log.debug('DB -> %s' % sql)
self.execute(sql)
def create_db(self, db_name):
"""
@brief Creates a database
@param db_name String
"""
if self.conf['database'] not in self.show_database():
sql = 'CREATE DATABASE IF NOT EXISTS %s CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci' % db_name
Log.debug('DB -> %s' % sql)
self.execute(sql)
def create_table(self, table, cols):
"""
@brief Creates a table in database
@param table String
@param cols String, the cols in table
"""
sql = "CREATE TABLE if not exists %s (%s);" % (table, cols)
Log.debug('DB -> %s' % sql)
self.execute(sql)
def close(self):
"""
@brief close connection to database
"""
Log.debug('DB -> close')
# 关闭数据库连接
self.conn.close()
def create_table(self, table, cols):
"""
@brief Creates a table in database
@param table String
@param cols String, the cols in table
"""
sql = "CREATE TABLE if not exists %s (%s);" % (table, cols)
Log.debug('DB -> %s' % sql)
self.execute(sql)
def insert(self, table, value):
"""
@brief Insert a row in table
@param table String
@param value Tuple
"""
sql = ("INSERT INTO %s VALUES (" + ",".join(['?'] * len(value)) + ");") % table
Log.debug('DB -> %s' % sql)
self.execute(sql, value)
def delete(self, table, field='', condition=''):
"""
@brief execute sql commands, return result if it has
@param table String
@param field String
@param condition String
"""
sql = "DELETE FROM %s WHERE %s=%s" % (table, field, condition)
Log.debug('DB -> %s' % sql)
self.execute(sql)
def insertmany(self, table, values):
"""
@brief Insert many rows in table
@param table String
@param values Array of tuple
"""
col_name = self.table_cols[table][1:]
sql = 'INSERT INTO %s(%s) VALUES (%s)' % (table, ','.join(col_name), ','.join(['%s'] * len(values[0])))
Log.debug('DB -> %s' % sql)
self.execute(sql, values)
def delete_table(self, table):
"""
@brief Delete a table in database
@param table String
"""
if table in self.table_cols:
sql = "DROP TABLE IF EXISTS %s" % table
Log.debug('DB -> %s' % sql)
self.execute(sql)
self.table_cols.pop(table)
def insert(self, table, value):
"""
@brief Insert a row in table
@param table String
@param value Tuple
"""
col_name = self.table_cols[table][1:]
sql = "INSERT INTO %s(%s) VALUES (%s)" % (table, str(','.join(col_name)), array_join(value, ','))
Log.debug('DB -> %s' % sql)
self.execute(sql)
def registerAgent(self, agent, method='random', **kwargs):
# Registering randomly
if method == 'random':
rand_x, rand_y = random.randint(0, self.width - 1), random.randint(
0, self.height - 1)
pos = GridMap.GridPosition(x=rand_x, y=rand_y)
self._agent_to_pos[agent] = pos
self._pos_to_agent[pos][agent] = agent
Log.debug("Registered new agent %s with position %s" %
(hex(id(agent)), str(pos)))
def delete_table(self, table):
"""
@brief Delete a table in database
@param table String
"""
if table in self.table_cols:
sql = "DROP TABLE IF EXISTS %s" % table
Log.debug('DB -> %s' % sql)
self.execute(sql)
self.table_cols.pop(table)
def delete(self, table, field='', condition=''):
"""
@brief execute sql commands, return result if it has
@param table String
@param field String
@param condition String
"""
sql = "DELETE FROM %s WHERE %s=%s" % (table, field, condition)
Log.debug('DB -> %s' % sql)
self.execute(sql)
def str2qr_terminal(text):
"""
@brief convert string to qrcode matrix and outprint
@param text The string
"""
Log.debug(text)
qr = qrcode.QRCode()
qr.border = 1
qr.add_data(text)
mat = qr.get_matrix()
print_qr(mat)
def str2qr_terminal(text):
"""
@brief convert string to qrcode matrix and outprint
@param text The string
"""
Log.debug(text)
qr = qrcode.QRCode()
qr.border = 1
qr.add_data(text)
mat = qr.get_matrix()
print_qr(mat)
def check_schedule_task(self):
# update group member list at 00:00 am every morning
t = time.localtime()
if t.tm_hour == 0 and t.tm_min <= 1:
# update group member
Log.debug('update group member list everyday')
self.db.delete_table(Constant.TABLE_GROUP_LIST())
self.db.delete_table(Constant.TABLE_GROUP_USER_LIST())
self.db.create_table(Constant.TABLE_GROUP_LIST(), Constant.TABLE_GROUP_LIST_COL)
self.db.create_table(Constant.TABLE_GROUP_USER_LIST(), Constant.TABLE_GROUP_USER_LIST_COL)
self.wechat.fetch_group_contacts()
def create_table(self, table, cols):
"""
@brief Creates a table in database
@param table String
@param cols String, the cols in table
"""
if table not in self.table_cols:
sql = 'CREATE TABLE IF NOT EXISTS %s(id int primary key auto_increment, %s) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci' % (table, cols)
Log.debug('DB -> %s' % sql)
self.execute(sql)
self.table_cols[table] = ['id'] + [c.strip().split(' ')[0] for c in cols.split(',')]
def insertmany(self, table, values):
"""
@brief Insert many rows in table
@param table String
@param values Array of tuple
"""
col_name = self.table_cols[table][1:]
sql = 'INSERT INTO %s(%s) VALUES (%s)' % (
table, ','.join(col_name), ','.join(['%s'] * len(values[0])))
Log.debug('DB -> %s' % sql)
self.execute(sql, values)
def insert(self, table, value):
"""
@brief Insert a row in table
@param table String
@param value Tuple
"""
col_name = self.table_cols[table][1:]
sql = "INSERT INTO %s(%s) VALUES (%s)" % (table, str(
','.join(col_name)), array_join(value, ','))
Log.debug('DB -> %s' % sql)
self.execute(sql)
def get_token(self):
login_data = {
'username': self.username,
'password': self.password,
'public_key': None
}
token = requests.post(url=self.url + '/api/v1/authentication/auth/',
data=login_data).text
Log.debug('token is %s' % token)
self.headers['Authorization'] = 'Bearer %s' % json.loads(
token)['token']
def get_aliyun_hosts(self,
register_prefix='',
default_re='',
extend_re='dev',
project=''):
client = AcsClient(self.accessKeyID, self.accessKeySecret, self.region)
request = DescribeInstancesRequest()
request.set_accept_format('json')
request.set_PageNumber(1)
request.set_PageSize(50)
response = client.do_action_with_exception(request)
data = json.loads(response)
PageNumber = int(data['TotalCount'] / 50 + 2)
list = []
search_shard = extend_mesage(project)
for Number in range(1, PageNumber):
request.set_PageNumber(Number)
request.set_PageSize(50)
response = client.do_action_with_exception(request)
data = json.loads(response)
for host in data['Instances']['Instance']:
dict = {}
id = re.split(r'-', host['SerialNumber'])
id = ''.join(str(i) for i in id)
dict['id'] = id
dict['ip'] = host['NetworkInterfaces']['NetworkInterface'][0][
'PrimaryIpAddress']
if register_prefix:
dict['hostname'] = str(
register_prefix) + host['InstanceName']
else:
dict['hostname'] = host['InstanceName']
dict['is_active'] = 'true'
dict['platform'] = "Linux"
try:
dict['comment'] = search_shard[dict['ip']]
except KeyError:
if 'Tags' in host:
comment = host['Tags']['Tag'][0]['TagValue']
dict['comment'] = "[%s]" % comment
else:
dict['comment'] = "[123]"
# if host['InstanceName'].startswith('G') or host['InstanceName'].startswith('c'):
final_re = '|'.join(
default_re.split('|') + extend_re.split('|')).strip('|')
if re.search(final_re, host['InstanceName']):
dict['group'] = ['dev']
else:
dict['group'] = ['base']
list.append(dict)
Log.debug(list)
return list
def select(self, table, field='', condition=''):
"""
@brief select all result from table
@param table String
@param field String
@param condition String
@return result Tuple
"""
sql = "SELECT * FROM %s" % table
if field and condition:
sql += " WHERE %s='%s'" % (field, condition)
Log.debug('DB -> %s' % sql)
return self.execute(sql)
def select(self, table, field='', condition=''):
"""
@brief select all result from table
@param table String
@param field String
@param condition String
@return result Tuple
"""
sql = "SELECT * FROM %s" % table
if field and condition:
sql += " WHERE %s='%s'" % (field, condition)
Log.debug('DB -> %s' % sql)
return self.execute(sql)
def check_schedule_task(self):
# update group member list at 00:00 am every morning
t = time.localtime()
if t.tm_hour == 0 and t.tm_min <= 1:
# update group member
Log.debug('update group member list everyday')
self.db.delete_table(Constant.TABLE_GROUP_LIST())
self.db.delete_table(Constant.TABLE_GROUP_USER_LIST())
self.db.create_table(Constant.TABLE_GROUP_LIST(),
Constant.TABLE_GROUP_LIST_COL)
self.db.create_table(Constant.TABLE_GROUP_USER_LIST(),
Constant.TABLE_GROUP_USER_LIST_COL)
self.wechat.fetch_group_contacts()
def update(self, table, dic, condition=''):
k_arr = []
v_arr = []
for (k, v) in dic.items():
k_arr.append('%s=?' % k)
v_arr.append(v)
sql = "UPDATE %s SET %s" % (table, ','.join(k_arr))
if condition:
sql += " WHERE %s" % condition
Log.debug('DB -> %s' % sql)
self.execute(sql, tuple(v_arr))
def update(self, table, dic, condition=''):
k_arr = []
v_arr = []
for (k, v) in dic.items():
k_arr.append('%s=?' % k)
v_arr.append(v)
sql = "UPDATE %s SET %s" % (table, ','.join(k_arr))
if condition:
sql += " WHERE %s" % condition
Log.debug('DB -> %s' % sql)
self.execute(sql, tuple(v_arr))
def create_table(self, table, cols):
"""
@brief Creates a table in database
@param table String
@param cols String, the cols in table
"""
if table not in self.table_cols:
sql = 'CREATE TABLE IF NOT EXISTS %s(id int primary key auto_increment, %s) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci' % (
table, cols)
Log.debug('DB -> %s' % sql)
self.execute(sql)
self.table_cols[table] = ['id'] + [
c.strip().split(' ')[0] for c in cols.split(',')
]
def assets_update(self, asset_info, jms_info):
jms_info_asset = [x for x in jms_info
if x['id'] == asset_info['id']][0]
if (jms_info_asset['hostname'] == asset_info['hostname']
and jms_info_asset['ip'] == asset_info['ip']
and jms_info_asset['comment'] == asset_info['comment']):
Log.debug('The assets %s is no change' % asset_info['hostname'])
else:
rst = requests.put(url=self.url +
'/api/v1/assets/assets/%s/' % asset_info['id'],
headers=self.headers,
data=json.dumps(asset_info))
Log.info('Update the assent %s: %s' %
(rst.status_code, asset_info['hostname']))
def get_assents(self,
register_prefix='',
default_re='',
extend_re='dev',
project=''):
final_re = '|'.join(default_re.split('|') +
extend_re.split('|')).strip('|')
Log.debug(final_re)
server_list = []
search_shard = extend_mesage(project)
for each_instance in self.ec2.instances.all():
server_info = {}
Name_index = [
i for i, x in enumerate(each_instance.tags)
if x['Key'].find('Name') != -1
]
if re.search(final_re, each_instance.tags[Name_index[0]]['Value']):
server_info['group'] = ['dev']
else:
server_info['group'] = ['base']
if register_prefix:
server_info['hostname'] = str(
register_prefix) + each_instance.tags[
Name_index[0]]['Value']
else:
server_info['hostname'] = each_instance.tags[
Name_index[0]]['Value']
server_info['id'] = str(
uuid.uuid3(uuid.NAMESPACE_DNS, each_instance.id))
server_info['ip'] = each_instance.private_ip_address
server_info['is_actice'] = 'true'
server_info['platform'] = "Linux"
try:
server_info['comment'] = search_shard[
each_instance.private_ip_address]
except KeyError:
Comman_index = [
i for i, x in enumerate(each_instance.tags)
if x['Key'].find('Service') != -1
]
if Comman_index:
comman_info = str(
each_instance.tags[Comman_index[0]]['Value'])
else:
comman_info = '[]'
server_info['comment'] = comman_info
Log.debug(server_info)
server_list.append(server_info)
return server_list
def extend_mesage(project_config):
a = Etcd(project_config)
db = a.get_data(multi=20)
rst = {}
for each_record in db:
Log.debug('Etcd each Record %s' % each_record)
ip = each_record['gm/private_ip']
try:
rst[ip] += ',%s' % each_record['merge_rel']
except KeyError:
rst[ip] = each_record['merge_rel']
'''
value need list uniq
'''
return rst
def fetch_group_contacts(self):
"""
@brief Fetches all groups contacts.
@return Bool: whether operation succeed.
@note This function must be finished in 180s
"""
Log.debug('fetch_group_contacts')
max_thread_num = 4
max_fetch_group_num = 50
group_list_queue = Queue.Queue()
class GroupListThread(threading.Thread):
def __init__(self, group_list_queue, wechat):
threading.Thread.__init__(self)
self.group_list_queue = group_list_queue
self.wechat = wechat
def run(self):
while not self.group_list_queue.empty():
gid_list = self.group_list_queue.get()
group_member_list = self.wechat.webwxbatchgetcontact(
gid_list)
for member_list in group_member_list[:]:
g_member_list = member_list['MemberList'][:]
member_list['MemberList'] = []
self.wechat.GroupList.append(member_list)
self.wechat.GroupMemeberList[
member_list['UserName']] = g_member_list
self.group_list_queue.task_done()
for g_list in split_array(self.addGroupIDList, max_fetch_group_num):
group_list_queue.put(g_list)
for i in range(max_thread_num):
t = GroupListThread(group_list_queue, self)
t.setDaemon(True)
t.start()
group_list_queue.join()
# 对于群成员的处理不使用数据库，直接在通过内存中的字典进行操作，
# 因为python对于字典的优化已经做的非常不错，通过数据库进行处理并不能显著提升性能，并没有必要
# 而且就时效性考虑，因为每次群ID都在变化，就算存入数据库中，在下次运行程序时依然要重新写入数据库
# 所以并不能节省开销
return True
def save_json(filename, data, dirName, mode='w+'):
"""
@brief Saves dict to json file.
@param filename String
@param data Dict
@param dirName String
@return file path
"""
Log.debug('save json: ' + filename)
fn = filename
if not os.path.exists(dirName):
os.makedirs(dirName)
fn = os.path.join(dirName, filename)
with open(fn, mode) as f:
f.write(json.dumps(data, indent=4)+'\n')
return fn
def save_file(filename, data, dirName):
"""
@brief Saves raw data to file.
@param filename String
@param data Binary data
@param dirName String
@return file path
"""
Log.debug('save file: ' + filename)
fn = filename
if not os.path.exists(dirName):
os.makedirs(dirName)
fn = os.path.join(dirName, filename)
with open(fn, 'wb') as f:
f.write(data)
return fn
def insertmany(self, table, values):
"""
@brief Insert many rows in table
@param table String
@param values Array of tuple
"""
c = self.conn.cursor()
self.lock.acquire()
n = len(values[0])
sql = ("INSERT INTO %s VALUES (" + ",".join(['?'] * n) + ");") % table
Log.debug('DB -> %s' % sql)
try:
c.executemany(sql, values)
except Exception, e:
Log.error(traceback.format_exc())
def save_json(filename, data, dirName, mode='w+'):
"""
@brief Saves dict to json file.
@param filename String
@param data Dict
@param dirName String
@return file path
"""
Log.debug('save json: ' + filename)
fn = filename
if not os.path.exists(dirName):
os.makedirs(dirName)
fn = os.path.join(dirName, filename)
with open(fn, mode) as f:
f.write(json.dumps(data, indent=4) + '\n')
return fn
def save_file(filename, data, dirName):
"""
@brief Saves raw data to file.
@param filename String
@param data Binary data
@param dirName String
@return file path
"""
Log.debug('save file: ' + filename)
fn = filename
if not os.path.exists(dirName):
os.makedirs(dirName)
fn = os.path.join(dirName, filename)
with open(fn, 'wb') as f:
f.write(data)
return fn
def insertmany(self, table, values):
"""
@brief Insert many rows in table
@param table String
@param values Array of tuple
"""
c = self.conn.cursor()
self.lock.acquire()
n = len(values[0])
sql = ("INSERT INTO %s VALUES (" + ",".join(['?'] * n) + ");") % table
Log.debug('DB -> %s' % sql)
try:
c.executemany(sql, values)
except Exception, e:
Log.error(traceback.format_exc())
def get_data(self, multi=1):
base_url = urljoin(self.etcd_host,
path.join('v2/keys', self.etcd_root.strip('/')))
Log.debug('baseUrl: %s' % base_url)
try:
root_all = []
sub_keys = json.loads(requests.get(base_url).content)
m_parten = '%s/[0-9]{1,3}$' % self.etcd_root
for each_keys in sub_keys['node']['nodes']:
rst = self.mul_gid(m_parten, each_keys, multi=multi)
if rst:
root_all += [i.get() for i in rst if i.get()]
return root_all
except IOError:
Log.error('Network is unavailable')
except KeyError:
Log.error('Etcd response no available key')
def mul_shard(self, each_shard):
shard_info = {}
i_parten = '%s/[0-9]{3}/[0-9]{4,5}$' % self.etcd_root
if re.search(i_parten, str(each_shard['key'])):
Log.debug('All shard info %s' % each_shard)
shard_url = path.join(self.etcd_host, 'v2/keys',
str(each_shard['key']).strip('/'))
info_get = ['merge_rel', 'gm/private_ip']
try:
for i in info_get:
rst_each = json.loads(
requests.get(path.join(shard_url, i)).content)
shard_info[i] = rst_each['node']['value']
except KeyError:
shard_info.clear()
Log.warning('No such key %s' % rst_each)
return shard_info
def select(self, table, field='', condition=''):
"""
@brief select all result from table
@param table String
@param field String
@param condition String
@return result Tuple
"""
result = []
if field and condition:
cond = (condition,)
sql = "SELECT * FROM %s WHERE %s=?" % (table, field)
Log.debug('DB -> %s' % sql)
result = self.execute(sql, cond)
else:
sql = "SELECT * FROM %s" % table
Log.debug('DB -> %s' % sql)
result = self.execute(sql)
return result
def select(self, table, field='', condition=''):
"""
@brief select all result from table
@param table String
@param field String
@param condition String
@return result Tuple
"""
result = []
if field and condition:
cond = (condition,)
sql = "SELECT * FROM %s WHERE %s=?" % (table, field)
Log.debug('DB -> %s' % sql)
result = self.execute(sql, cond)
else:
sql = "SELECT * FROM %s" % table
Log.debug('DB -> %s' % sql)
result = self.execute(sql)
return result
def handle_mod(self, r):
# ModContactCount: 变更联系人或群聊成员数目
# ModContactList: 变更联系人或群聊列表，或群名称改变
Log.debug('handle modify')
self.handle_msg(r)
for m in r['ModContactList']:
if m['UserName'][:2] == '@@':
# group
in_list = False
g_id = m['UserName']
for g in self.GroupList:
# group member change
if g_id == g['UserName']:
g['MemberCount'] = m['MemberCount']
g['NickName'] = m['NickName']
self.GroupMemeberList[g_id] = m['MemberList']
in_list = True
if self.msg_handler:
self.msg_handler.handle_group_member_change(
g_id, m['MemberList'])
break
if not in_list:
# a new group
self.GroupList.append(m)
self.GroupMemeberList[g_id] = m['MemberList']
if self.msg_handler:
self.msg_handler.handle_group_list_change(m)
self.msg_handler.handle_group_member_change(
g_id, m['MemberList'])
elif m['UserName'][0] == '@':
# user
in_list = False
for u in self.MemberList:
u_id = m['UserName']
if u_id == u['UserName']:
u = m
in_list = True
break
# if don't have then add it
if not in_list:
self.MemberList.append(m)
def handle_mod(self, r):
# ModContactCount: 变更联系人或群聊成员数目
# ModContactList: 变更联系人或群聊列表，或群名称改变
Log.debug('handle modify')
self.handle_msg(r)
for m in r['ModContactList']:
if m['UserName'][:2] == '@@':
# group
in_list = False
g_id = m['UserName']
for g in self.GroupList:
# group member change
if g_id == g['UserName']:
g['MemberCount'] = m['MemberCount']
g['NickName'] = m['NickName']
self.GroupMemeberList[g_id] = m['MemberList']
in_list = True
if self.msg_handler:
self.msg_handler.handle_group_member_change(g_id, m['MemberList'])
break
if not in_list:
# a new group
self.GroupList.append(m)
self.GroupMemeberList[g_id] = m['MemberList']
if self.msg_handler:
self.msg_handler.handle_group_list_change(m)
self.msg_handler.handle_group_member_change(g_id, m['MemberList'])
elif m['UserName'][0] == '@':
# user
in_list = False
for u in self.MemberList:
u_id = m['UserName']
if u_id == u['UserName']:
u = m
in_list = True
break
# if don't have then add it
if not in_list:
self.MemberList.append(m)
def snapshot(self):
"""
@brief Save basic infos for next login.
@return Bool: whether operation succeed.
"""
try:
conf = {
'uuid': self.uuid,
'redirect_uri': self.redirect_uri,
'uin': self.uin,
'sid': self.sid,
'skey': self.skey,
'pass_ticket': self.pass_ticket,
'synckey': self.synckey,
'device_id': self.device_id,
'last_login': time.time(),
}
cm = ConfigManager()
Log.debug('save wechat config')
cm.set_wechat_config(conf)
# save cookie
Log.debug('save cookie')
if self.cookie:
self.cookie.save(ignore_discard=True)
# save contacts
Log.debug('save contacts')
self.save_contacts()
except Exception, e:
Log.error(traceback.format_exc())
return False
def post(url, params, jsonfmt=True):
"""
@brief http post request
@param url String
@param params Dict, post params
@param jsonfmt Bool, whether is json format
@return http response
"""
Log.debug('POST -> ' + url)
Log.debug(params)
if jsonfmt:
request = urllib2.Request(
url=url,
data=json.dumps(params, ensure_ascii=False).encode('utf8'))
request.add_header(*Constant.HTTP_HEADER_CONTENTTYPE)
else:
request = urllib2.Request(url=url, data=urllib.urlencode(params))
while True:
try:
response = urllib2.urlopen(request, timeout=30)
data = response.read()
response.close()
if jsonfmt:
Log.debug(data)
return json.loads(data, object_hook=_decode_data)
return data
except (KeyboardInterrupt, SystemExit):
raise
except:
Log.error(traceback.format_exc())
time.sleep(1)
def post(url, params, jsonfmt=True):
"""
@brief http post request
@param url String
@param params Dict, post params
@param jsonfmt Bool, whether is json format
@return http response
"""
Log.debug('POST -> '+url)
Log.debug(params)
if jsonfmt:
request = urllib2.Request(url=url, data=json.dumps(params, ensure_ascii=False).encode('utf8'))
request.add_header(*Constant.HTTP_HEADER_CONTENTTYPE)
else:
request = urllib2.Request(url=url, data=urllib.urlencode(params))
while True:
try:
response = urllib2.urlopen(request, timeout=30)
data = response.read()
response.close()
if jsonfmt:
Log.debug(data)
return json.loads(data, object_hook=_decode_data)
return data
except (KeyboardInterrupt, SystemExit):
raise
except:
Log.error(traceback.format_exc())
time.sleep(1)
def snapshot(self):
"""
@brief Save basic infos for next login.
@return Bool: whether operation succeed.
"""
try:
conf = {
'uuid': self.uuid,
'redirect_uri': self.redirect_uri,
'uin': self.uin,
'sid': self.sid,
'skey': self.skey,
'pass_ticket': self.pass_ticket,
'synckey': self.synckey,
'device_id': self.device_id,
'last_login': time.time(),
}
cm = ConfigManager()
Log.debug('save wechat config')
cm.set_wechat_config(conf)
# save cookie
Log.debug('save cookie')
if self.cookie:
self.cookie.save(ignore_discard=True)
# save contacts
Log.debug('save contacts')
self.save_contacts()
except Exception, e:
Log.error(traceback.format_exc())
return False
def show_tables(self):
c = self.conn.cursor()
sql = 'SHOW TABLES'
Log.debug('DB -> %s' % sql)
c.execute(sql)
return [r['Tables_in_'+self.conf['database']] for r in c.fetchall()]
def fetch_group_contacts(self):
"""
@brief Fetches all groups contacts.
@return Bool: whether operation succeed.
@note This function must be finished in 180s
"""
Log.debug('fetch_group_contacts')
# clean database
if self.msg_handler:
self.msg_handler.clean_db()
# sqlite
# ----------------------------------------------------
# group max_thread_num max_fetch_group_num time(s)
# 197 10 10 108
# 197 10 15 95
# 197 20 10 103
# 197 10 20 55
# 197 5 30 39
# 197 4 50 35
# ----------------------------------------------------
# mysql
# ----------------------------------------------------
# group max_thread_num max_fetch_group_num time(s)
# 197 4 50 20
# ----------------------------------------------------
max_thread_num = 4
max_fetch_group_num = 50
group_list_queue = Queue.Queue()
class GroupListThread(threading.Thread):
def __init__(self, group_list_queue, wechat):
threading.Thread.__init__(self)
self.group_list_queue = group_list_queue
self.wechat = wechat
def run(self):
while not self.group_list_queue.empty():
g_list = self.group_list_queue.get()
gid_list = []
g_dict = {}
for g in g_list:
gid = g['UserName']
gid_list.append(gid)
g_dict[gid] = g
group_member_list = self.wechat.webwxbatchgetcontact(gid_list)
for member_list in group_member_list:
gid = member_list['UserName']
g = g_dict[gid]
g['MemberCount'] = member_list['MemberCount']
g['OwnerUin'] = member_list['OwnerUin']
self.wechat.GroupMemeberList[gid] = member_list['MemberList']
# 如果使用 Mysql 则可以在多线程里操作数据库
# 否则请注释下列代码在主线程里更新群列表
# -----------------------------------
# 处理群成员
# if self.wechat.msg_handler:
# self.wechat.msg_handler.handle_group_member_list(gid, member_list['MemberList'])
# -----------------------------------
self.group_list_queue.task_done()
for g_list in split_array(self.GroupList, max_fetch_group_num):
group_list_queue.put(g_list)
for i in range(max_thread_num):
t = GroupListThread(group_list_queue, self)
t.setDaemon(True)
t.start()
group_list_queue.join()
if self.msg_handler:
# 处理群
if self.GroupList:
self.msg_handler.handle_group_list(self.GroupList)
# 这个是用 sqlite 来存储群列表，sqlite 对多线程的支持不太好
# ----------------------------------------------------
# 处理群成员
for (gid, member_list) in self.GroupMemeberList.items():
self.msg_handler.handle_group_member_list(gid, member_list)
# ----------------------------------------------------
return True
def handle_msg(self, r):
"""
@brief Recover from snapshot data.
@param r Dict: message json
"""
Log.debug('handle message')
if self.msg_handler:
self.msg_handler.handle_wxsync(r)
n = len(r['AddMsgList'])
if n == 0:
return
if self.log_mode:
echo(Constant.LOG_MSG_NEW_MSG % n)
for msg in r['AddMsgList']:
msgType = msg['MsgType']
msgId = msg['MsgId']
content = msg['Content'].replace('<', '<').replace('>', '>')
raw_msg = None
if msgType == self.wx_conf['MSGTYPE_TEXT']:
# 地理位置消息
if content.find('pictype=location') != -1:
location = content.split('<br/>')[1][:-1]
raw_msg = {
'raw_msg': msg,
'location': location,
'log': Constant.LOG_MSG_LOCATION % location
}
# 普通文本消息
else:
text = content.split(':<br/>')[-1]
raw_msg = {
'raw_msg': msg,
'text': text,
'log': text.replace('<br/>', '\n')
}
elif msgType == self.wx_conf['MSGTYPE_IMAGE']:
data = self.webwxgetmsgimg(msgId)
fn = 'img_' + msgId + '.jpg'
dir = self.save_data_folders['webwxgetmsgimg']
path = save_file(fn, data, dir)
raw_msg = {'raw_msg': msg,
'image': path,
'log': Constant.LOG_MSG_PICTURE % path}
elif msgType == self.wx_conf['MSGTYPE_VOICE']:
data = self.webwxgetvoice(msgId)
fn = 'voice_' + msgId + '.mp3'
dir = self.save_data_folders['webwxgetvoice']
path = save_file(fn, data, dir)
raw_msg = {'raw_msg': msg,
'voice': path,
'log': Constant.LOG_MSG_VOICE % path}
elif msgType == self.wx_conf['MSGTYPE_SHARECARD']:
info = msg['RecommendInfo']
card = Constant.LOG_MSG_NAME_CARD % (
info['NickName'],
info['Alias'],
info['Province'], info['City'],
Constant.LOG_MSG_SEX_OPTION[info['Sex']]
)
namecard = '%s %s %s %s %s' % (
info['NickName'], info['Alias'], info['Province'],
info['City'], Constant.LOG_MSG_SEX_OPTION[info['Sex']]
)
raw_msg = {
'raw_msg': msg,
'namecard': namecard,
'log': card
}
elif msgType == self.wx_conf['MSGTYPE_EMOTICON']:
url = search_content('cdnurl', content)
raw_msg = {'raw_msg': msg,
'emoticon': url,
'log': Constant.LOG_MSG_EMOTION % url}
elif msgType == self.wx_conf['MSGTYPE_APP']:
card = ''
# 链接, 音乐, 微博
if msg['AppMsgType'] in [
self.wx_conf['APPMSGTYPE_AUDIO'],
self.wx_conf['APPMSGTYPE_URL'],
self.wx_conf['APPMSGTYPE_OPEN']
]:
card = Constant.LOG_MSG_APP_LINK % (
Constant.LOG_MSG_APP_LINK_TYPE[msg['AppMsgType']],
msg['FileName'],
search_content('des', content, 'xml'),
msg['Url'],
search_content('appname', content, 'xml')
)
raw_msg = {
'raw_msg': msg,
'link': msg['Url'],
'log': card
}
# 图片
elif msg['AppMsgType'] == self.wx_conf['APPMSGTYPE_IMG']:
data = self.webwxgetmsgimg(msgId)
fn = 'img_' + msgId + '.jpg'
dir = self.save_data_folders['webwxgetmsgimg']
path = save_file(fn, data, dir)
card = Constant.LOG_MSG_APP_IMG % (
path,
search_content('appname', content, 'xml')
)
raw_msg = {
'raw_msg': msg,
'image': path,
'log': card
}
else:
raw_msg = {
'raw_msg': msg,
'log': Constant.LOG_MSG_UNKNOWN_MSG % (msgType, content)
}
elif msgType == self.wx_conf['MSGTYPE_STATUSNOTIFY']:
Log.info(Constant.LOG_MSG_NOTIFY_PHONE)
elif msgType == self.wx_conf['MSGTYPE_MICROVIDEO']:
data = self.webwxgetvideo(msgId)
fn = 'video_' + msgId + '.mp4'
dir = self.save_data_folders['webwxgetvideo']
path = save_file(fn, data, dir)
raw_msg = {'raw_msg': msg,
'video': path,
'log': Constant.LOG_MSG_VIDEO % path}
elif msgType == self.wx_conf['MSGTYPE_RECALLED']:
recall_id = search_content('msgid', content, 'xml')
text = Constant.LOG_MSG_RECALL
raw_msg = {
'raw_msg': msg,
'text': text,
'recall_msg_id': recall_id,
'log': text
}
elif msgType == self.wx_conf['MSGTYPE_SYS']:
raw_msg = {
'raw_msg': msg,
'sys_notif': content,
'log': content
}
elif msgType == self.wx_conf['MSGTYPE_VERIFYMSG']:
name = search_content('fromnickname', content)
raw_msg = {
'raw_msg': msg,
'log': Constant.LOG_MSG_ADD_FRIEND % name
}
else:
raw_msg = {
'raw_msg': msg,
'log': Constant.LOG_MSG_UNKNOWN_MSG % (msgType, content)
}
isGroupMsg = '@@' in msg['FromUserName']+msg['ToUserName']
if self.msg_handler and raw_msg:
if isGroupMsg:
# handle group messages
g_msg = self.make_group_msg(raw_msg)
self.msg_handler.handle_group_msg(g_msg)
else:
# handle personal messages
self.msg_handler.handle_user_msg(raw_msg)
if self.log_mode:
self.show_msg(raw_msg)
def make_group_msg(self, msg):
"""
@brief Package the group message for storage.
@param msg Dict: raw msg
@return raw_msg Dict: packged msg
"""
Log.debug('make group message')
raw_msg = {
'raw_msg': msg['raw_msg'],
'msg_id': msg['raw_msg']['MsgId'],
'group_owner_uin': '',
'group_name': '',
'group_count': '',
'from_user_name': msg['raw_msg']['FromUserName'],
'to_user_name': msg['raw_msg']['ToUserName'],
'user_attrstatus': '',
'user_display_name': '',
'user_nickname': '',
'msg_type': msg['raw_msg']['MsgType'],
'text': '',
'link': '',
'image': '',
'video': '',
'voice': '',
'emoticon': '',
'namecard': '',
'location': '',
'recall_msg_id': '',
'sys_notif': '',
'time': '',
'timestamp': '',
'log': '',
}
content = msg['raw_msg']['Content'].replace(
'<', '<').replace('>', '>')
group = None
src = None
if msg['raw_msg']['FromUserName'][:2] == '@@':
# 接收到来自群的消息
g_id = msg['raw_msg']['FromUserName']
group = self.get_group_by_id(g_id)
if re.search(":<br/>", content, re.IGNORECASE):
u_id = content.split(':<br/>')[0]
src = self.get_group_user_by_id(u_id, g_id)
elif msg['raw_msg']['ToUserName'][:2] == '@@':
# 自己发给群的消息
g_id = msg['raw_msg']['ToUserName']
u_id = msg['raw_msg']['FromUserName']
src = self.get_group_user_by_id(u_id, g_id)
group = self.get_group_by_id(g_id)
if src:
raw_msg['user_attrstatus'] = src['AttrStatus']
raw_msg['user_display_name'] = src['DisplayName']
raw_msg['user_nickname'] = src['NickName']
if group:
raw_msg['group_count'] = group['MemberCount']
raw_msg['group_owner_uin'] = group['OwnerUin']
raw_msg['group_name'] = group['ShowName']
raw_msg['timestamp'] = msg['raw_msg']['CreateTime']
t = time.localtime(float(raw_msg['timestamp']))
raw_msg['time'] = time.strftime("%Y-%m-%d %T", t)
for key in [
'text', 'link', 'image', 'video', 'voice',
'emoticon', 'namecard', 'location', 'log',
'recall_msg_id', 'sys_notif'
]:
if key in msg:
raw_msg[key] = msg[key]
return raw_msg
def start(self):
echo(Constant.LOG_MSG_START)
run(Constant.LOG_MSG_RECOVER, self.recover)
timeOut = time.time() - self.last_login
echo(Constant.LOG_MSG_TRY_INIT)
if self.webwxinit():
echo(Constant.LOG_MSG_SUCCESS)
run(Constant.LOG_MSG_RECOVER_CONTACT, self.recover_contacts)
else:
echo(Constant.LOG_MSG_FAIL)
while True:
# first try to login by uin without qrcode
echo(Constant.LOG_MSG_ASSOCIATION_LOGIN)
if self.association_login():
echo(Constant.LOG_MSG_SUCCESS)
else:
echo(Constant.LOG_MSG_FAIL)
# scan qrcode to login
run(Constant.LOG_MSG_GET_UUID, self.getuuid)
echo(Constant.LOG_MSG_GET_QRCODE)
self.genqrcode()
echo(Constant.LOG_MSG_SCAN_QRCODE)
if not self.waitforlogin():
continue
echo(Constant.LOG_MSG_CONFIRM_LOGIN)
if not self.waitforlogin(0):
continue
break
run(Constant.LOG_MSG_LOGIN, self.login)
run(Constant.LOG_MSG_INIT, self.webwxinit)
run(Constant.LOG_MSG_STATUS_NOTIFY, self.webwxstatusnotify)
run(Constant.LOG_MSG_GET_CONTACT, self.webwxgetcontact)
echo(Constant.LOG_MSG_CONTACT_COUNT % (
self.MemberCount, len(self.MemberList)
))
echo(Constant.LOG_MSG_OTHER_CONTACT_COUNT % (
len(self.GroupList), len(self.ContactList),
len(self.SpecialUsersList), len(self.PublicUsersList)
))
run(Constant.LOG_MSG_GET_GROUP_MEMBER, self.fetch_group_contacts)
run(Constant.LOG_MSG_SNAPSHOT, self.snapshot)
while True:
[retcode, selector] = self.synccheck()
Log.debug('retcode: %s, selector: %s' % (retcode, selector))
self.exit_code = int(retcode)
if retcode == '1100':
echo(Constant.LOG_MSG_LOGOUT)
break
if retcode == '1101':
echo(Constant.LOG_MSG_LOGIN_OTHERWHERE)
break
if retcode == '1102':
echo(Constant.LOG_MSG_QUIT_ON_PHONE)
break
elif retcode == '0':
if selector == '2':
r = self.webwxsync()
if r is not None:
try:
self.handle_msg(r)
except:
Log.error(traceback.format_exc())
elif selector == '7':
r = self.webwxsync()
elif selector == '0':
time.sleep(self.time_out)
elif selector == '4':
# 保存群聊到通讯录
# 修改群名称
# 新增或删除联系人
# 群聊成员数目变化
r = self.webwxsync()
if r is not None:
try:
self.handle_mod(r)
except:
Log.error(traceback.format_exc())
elif selector == '3' or selector == '6':
break
else:
r = self.webwxsync()
Log.debug('webwxsync: %s\n' % json.dumps(r))
# 执行定时任务
if self.msg_handler:
self.msg_handler.check_schedule_task()
def handle_group_msg(self, msg):
"""
@brief Recieve group messages
@param msg Dict: packaged msg
"""
# rename media files
for k in ['image', 'video', 'voice']:
if msg[k]:
t = time.localtime(float(msg['timestamp']))
time_str = time.strftime("%Y%m%d%H%M%S", t)
# format: 时间_消息ID_群名
file_name = '/%s_%s_%s.' % (time_str, msg['msg_id'], msg['group_name'])
new_name = re.sub(r'\/\w+\_\d+\.', file_name, msg[k])
Log.debug('rename file to %s' % new_name)
os.rename(msg[k], new_name)
msg[k] = new_name
if msg['msg_type'] == 10000:
# record member enter in group
m = re.search(r'邀请(.+)加入了群聊', msg['sys_notif'])
if m:
name = m.group(1)
col_enter_group = (
msg['msg_id'],
msg['group_name'],
msg['from_user_name'],
msg['to_user_name'],
name,
msg['time'],
)
self.db.insert(Constant.TABLE_RECORD_ENTER_GROUP, col_enter_group)
# record rename group
n = re.search(r'(.+)修改群名为“(.+)”', msg['sys_notif'])
if n:
people = n.group(1)
to_name = n.group(2)
col_rename_group = (
msg['msg_id'],
msg['group_name'],
to_name,
people,
msg['time'],
)
self.db.insert(Constant.TABLE_RECORD_RENAME_GROUP, col_rename_group)
# upadte group in GroupList
for g in self.wechat.GroupList:
if g['UserName'] == msg['from_user_name']:
g['NickName'] = to_name
break
# normal group message
col = (
msg['msg_id'],
msg['group_owner_uin'],
msg['group_name'],
msg['group_count'],
msg['from_user_name'],
msg['to_user_name'],
msg['user_attrstatus'],
msg['user_display_name'],
msg['user_nickname'],
msg['msg_type'],
msg['emoticon'],
msg['text'],
msg['image'],
msg['video'],
msg['voice'],
msg['link'],
msg['namecard'],
msg['location'],
msg['recall_msg_id'],
msg['sys_notif'],
msg['time'],
msg['timestamp']
)
self.db.insert(Constant.TABLE_GROUP_MSG_LOG, col)
text = msg['text']
if text and text[0] == '@':
n = trans_coding(text).find(u'\u2005')
name = trans_coding(text)[1:n].encode('utf-8')
if name in [self.wechat.User['NickName'], self.wechat.User['RemarkName']]:
self.handle_command(trans_coding(text)[n+1:].encode('utf-8'), msg)
def load_json(filepath):
Log.debug('load json: ' + filepath)
with open(filepath, 'r') as f:
return _decode_data(json.loads(f.read()))
