def avatar_request(self, login): """The function of requesting the avatar of the client from the server.""" LOGGER.debug(f'Avatar request for {login}') request = { ACTION: GET_AVATAR, ACCOUNT_NAME: login } with LOCK_SOCKET: try: send_msg(self.connection, request) answer = get_msg(self.connection) except (OSError, json.JSONDecodeError): return None if RESPONSE in answer and answer[RESPONSE] == 511: LOGGER.debug(f'Loaded avatar for {login}') img = answer[DATA] img_data = base64.b64decode(img) filename = get_path_avatar(login) with open(filename, 'wb') as f: f.write(img_data) return True elif RESPONSE in answer and answer[RESPONSE] == 400: LOGGER.info(f'Answer server {answer[ERROR]}') else: LOGGER.error(f'Failed to get the avatar {login}. ' f'Answer server {answer}')
def send_group_message(self, message): for client in self.names: if client != message[FROM]: try: send_msg(self.names[client], message) except OSError: self.remove_client(self.names[client])
def checking_new_client(self, client, message): if message[USER][ACCOUNT_NAME] in self.names.keys(): response = RESPONSE_400 response[ERROR] = 'Login already taken.' try: send_msg(client, response) except OSError: pass self.clients.remove(client) client.close() LOGGER.debug( f'Username is already taken. Response sent to client - {response} \n' ) elif not self.database.is_user(message[USER][ACCOUNT_NAME]): response = RESPONSE_400 response[ERROR] = 'User not registered.' try: send_msg(client, response) except OSError: pass self.clients.remove(client) client.close() LOGGER.debug( f'The user is not registered. Response sent to client - {response} \n' ) else: self.start_client_authorization(client, message)
def exit_client(self): try: send_msg(self.connection, self.create_exit_message(self.client_login)) except ConnectionResetError: LOGGER.critical('Lost server connection.') exit(1) LOGGER.info('Application shutdown by user command\n.') print('Application shutdown by user command.')
def _get_info(self, request): LOGGER.debug(f'Formed request {request}') with LOCK_SOCKET: send_msg(self.connection, request) answer = get_msg(self.connection) if RESPONSE in answer and answer[RESPONSE] == 202: return answer[LIST_INFO] else: raise ServerError('Invalid server response.')
def send_hash_password(self, answer_all): answer_data = answer_all[DATA] password_hash_string = self.get_hash_password() hash = hmac.new(password_hash_string, answer_data.encode('utf-8')) digest = hash.digest() my_answer = RESPONSE_511 my_answer[DATA] = binascii.b2a_base64(digest).decode('ascii') send_msg(self.connection, my_answer)
def send_groups(self): for client in self.names: try: msg = RESPONSE_206 msg[LIST_INFO] = [ group[1] for group in self.database.get_groups() ] send_msg(self.names[client], msg) except OSError: self.remove_client(self.names[client])
def update_users_list_message(self): """A method that implements sending a service message to 205 clients.""" for client in self.names: try: msg = RESPONSE_205 msg[LIST_INFO] = [ user[0] for user in self.database.users_all() ] send_msg(self.names[client], msg) except OSError: self.remove_client(self.names[client])
def shard_action(driver, udid): """ 分布式多机并行处理,数据分片 1、建立设备udid和索引的映射关系 2、根据数据库ID和索引分发任务到不同的设备上 :param driver: :param udid: :return: """ logging.info("【自动获取文章内容和用户评论】") # 建立索引 devices_list = glv.get('devices') idx = {} i = 0 for device in devices_list: idx[device] = i i = i + 1 # 进入对话框 utils.enter_talkbox(driver, 'com.tencent.mm:id/b4m') result = dbutil.shard_content_url(conn) logging.info('返回结果条数' + str(len(result))) for (id, url) in result: # 数据分片 if id % len(devices_list) != idx[udid]: continue # index = url.find('scene') # if index != -1: # url = url[0:index - 1] logging.info('采集文章: ' + str(id) + " " + url) if len(url) == 0: logging.info('跳过长度为0的url') continue # 发送消息 utils.send_msg(driver, url) # 点击链接 utils.click_last_msg_in_talkbox(driver, 'com.tencent.mm:id/nl') # 获取文章内容和评论并写入数据库 html = get_content_and_comment(driver) # 在网络延时下提取源码出错,只有122KB,故不更新放到下次重新提取 # logging.info('源码长度:%d' % len(html)) # if html and len(html) > 122000: if html: parse_html(str(url), html) # url_encode = quote(str(url), safe='') # URL编码 # name = os.path.join(base, url_encode) # utils.write_page_soure(name, html) # html = conn.escape_string(html) # dbutil.insert_content(conn, html, url) driver.quit() logging.info('共提取 ' + str(len(result) / len(devices_list)) + ' 篇文章')
def del_contact_server(self, del_contact_name): message = { ACTION: DELETE_CONTACT, TIME: time.time(), USER: self.client_login, ACCOUNT_NAME: del_contact_name } with LOCK_SOCKET: send_msg(self.connection, message) answer = get_msg(self.connection) if RESPONSE in answer and answer[RESPONSE] == 200: logging.debug(f'Successfully delete a contact {del_contact_name} at the user {self.client_login}') else: raise ServerError('Client uninstall error.')
def add_contact_server(self, new_contact_name): LOGGER.debug(f'Create a new contact {new_contact_name} at the user {self.client_login}.') message = { ACTION: ADD_CONTACT, TIME: time.time(), USER: self.client_login, ACCOUNT_NAME: new_contact_name } with LOCK_SOCKET: send_msg(self.connection, message) answer = get_msg(self.connection) if RESPONSE in answer and answer[RESPONSE] == 200: logging.debug(f'Successful contact creation {new_contact_name} at the user {self.client_login}.') else: raise ServerError('Error creating contact.')
def shard_action(driver, udid): logging.info("【开始自动获取公众号所有的历史消息】") # 该历史接口每天能访问200-300次,采集二十个,不能再多 start_id = 0 end_id = 0 task_file = "/Users/liushinan/PycharmProjects/wechatspider/configs/task.csv" data = autocontact.read_cofig(CONFIG_FILE) for device in data: if (device['udid']) == udid: start_id = device['start'] end_id = device['end'] break # 进入对话框 logging.info('start:' + str(start_id)) logging.info('end:' + str(end_id)) utils.enter_talkbox(driver, 'com.tencent.mm:id/b4m') count = 0 # 接口访问计数 with open(task_file, 'r', encoding="utf8", errors="ignore") as f: for line in f.readlines()[start_id:end_id]: line = line.replace("\n", "").split(",") id = line[0] biz = line[1] logging.info('----------------当前测试序号:' + str(id) + ' 当前测试biz:' + str(biz).lstrip('__biz=')) offset = 0 can_continue = True while can_continue and offset <= 130: count += 1 bizurl = 'https://mp.weixin.qq.com/mp/profile_ext?action=getmsg&__biz=' \ + biz + '&f=json&offset=' + str(offset) + '&count=10' + '\n' + id # 发送消息 utils.send_msg(driver, bizurl) # 点击链接 utils.click_last_msg_in_talkbox(driver, 'com.tencent.mm:id/nl') # 获取文章url can_continue = get_article(driver, biz) offset += 10 logging.info('下一个偏移量:%s' % offset) # 写入数据库 logging.info('接口总访问量:%s' % count) driver.quit() # 更新配置文件 start_id = end_id end_id = end_id + 15 # 写入 yaml 文件 autocontact.update_serial(CONFIG_FILE, udid, start_id, end_id)
async def send_message_user(self, msg): """Function respond to users.""" if msg[TO] in self.names and self.names[ msg[TO]] in self.clients_send_lst: send_msg(self.names[msg[TO]], msg) self.database.sending_message(msg[FROM], msg[TO]) LOGGER.info( f'A message was sent to user {msg [TO]} from user {msg [FROM]}.' ) elif msg[TO] in self.names and self.names[ msg[TO]] not in self.clients_send_lst: raise ConnectionError else: LOGGER.error( f'User {msg [TO]} is not registered on the server, sending messages is not possible.' )
def send_avatar_to_server(self): with open(get_path_avatar(self.client_login), 'rb') as image_file: encoded_img = base64.b64encode(image_file.read()).decode('utf8') message = { ACTION: SEND_AVATAR, USER: { ACCOUNT_NAME: self.client_login, IMAGE: encoded_img } } with LOCK_SOCKET: send_msg(self.connection, message) answer = get_msg(self.connection) if RESPONSE in answer and answer[RESPONSE] == 200: logging.debug(f'Successfully saved avatar.') else: raise ServerError('Server error. Unsuccessfully saved avatar.')
def shard_action(driver): print("【开始执行,每天自动关注分类公众号】") sql = 'SELECT id, biz from bizinfo WHERE addcontact=0 and id <= 80' # 新闻类 official_accounts = dbutil.query(conn, sql) # 进入对话框 utils.enter_talkbox(driver, 'com.tencent.mm:id/b4m') for (id, biz) in official_accounts: bizurl = 'https://mp.weixin.qq.com/mp/profile_ext?action=home&__biz=' \ + str(biz) + '&scene=124#wechat_redirect' + '\n' + str(id) # 发送消息 utils.send_msg(driver, bizurl) # 点击链接 utils.click_last_msg_in_talkbox(driver, 'com.tencent.mm:id/nl') # 点击自动关注 add_contact(driver) dbutil.update_bizinfo_addcontact(conn, biz) conn.close()
def shard_action(driver): logging.info("【开始自动获取公众号所有的历史消息】") sql = 'SELECT biz,nickname,history_offset from bizinfo WHERE spider=0 and id between 1 and 58' official_accounts = dbutil.query(conn, sql) # 进入对话框 utils.enter_talkbox(driver, 'com.tencent.mm:id/b4m') count = 0 # 接口访问计数 logging.info('查询总量:%d' % len(official_accounts)) outter_break = False for (biz, nickname, offset) in official_accounts: if outter_break or count > 180: logging.info('接口总访问量:%d' % count) break time_start = time.time() logging.info('----------------当前测试biz:' + str(biz)) logging.info('----------------当前测试nickname:' + str(nickname)) # 爬取半年之内的 while count <= 180: count += 1 bizurl = 'https://mp.weixin.qq.com/mp/profile_ext?action=getmsg&__biz=' \ + biz + '&f=json&offset=' + str(offset) + '&count=10' + '\n' + str(offset) # 发送消息 utils.send_msg(driver, bizurl) # 点击链接 utils.click_last_msg_in_talkbox(driver, 'com.tencent.mm:id/nl') # 获取文章url can_continue = get_article(driver, biz) # 更新偏移量 offset += 10 sql = "update bizinfo set history_offset= '{}' where biz= '{}'".format( offset, biz) dbutil.exec_sql(conn, sql) logging.info('下一个偏移量:%s' % offset) if can_continue == 'cannot_continue': time_end = time.time() sum_time = int(time_end - time_start) logging.info('单个公众号采集历史消息花费时间:%s' % str(sum_time)) dbutil.update_bizinfo_consume(conn, sum_time, biz) break elif can_continue == 'banned': outter_break = True break driver.quit()
def shard_action(driver, udid): logging.info("【开始自动获取公众号所有的历史消息】") articldata = Bizinfo() # 该历史接口每天能访问200-300次,采集二十个,不能再多 start_id = 0 end_id = 0 task_file = "/Users/liushinan/PycharmProjects/wechatspider/configs/task.csv" data = autocontact.read_cofig(CONFIG_FILE) for device in data: if (device['udid']) == udid: start_id = device['start'] end_id = device['end'] break # 进入对话框 logging.info('start:' + str(start_id)) logging.info('end:' + str(end_id)) utils.enter_talkbox(driver, 'com.tencent.mm:id/b4m') with open(task_file, 'r', encoding="utf8", errors="ignore") as f: for line in f.readlines()[start_id:end_id]: line = line.replace("\n", "").split(",") id = line[0] biz = line[1] logging.info('----------------当前测试序号:' + str(id) + ' 当前测试biz:' + str(biz).lstrip('__biz=')) bizurl = 'https://mp.weixin.qq.com/mp/profile_ext?action=home&__biz=' \ + biz + '&scene=124#wechat_redirect' + '\n' + id # 发送消息 utils.send_msg(driver, bizurl) # 点击链接 utils.click_last_msg_in_talkbox(driver, 'com.tencent.mm:id/nl') # 获取文章url articldata.get_article(driver, biz) # logging.info('所有数据:'+str(articldata.biz_info)) # 写入数据库 writeToDB(articldata.biz_info) driver.quit() # 更新配置文件 start_id = end_id end_id = end_id + 20 # 写入 yaml 文件 autocontact.update_serial(CONFIG_FILE, udid, start_id, end_id)
def shard_action(driver, udid): # 每天关注100个 print("【开始执行,每天自动关注180个公众号】") start_id = 0 end_id = 0 data = read_cofig(CONFIG_FILE) for device in data: if (device['udid']) == udid: start_id = device['start'] end_id = device['end'] break print('开始序列号' + str(start_id)) print('结束序列号' + str(end_id)) task_file = "/Users/liushinan/PycharmProjects/wechatspider/configs/task.csv" contacts = [] # 已关注的公众号 utils.enter_talkbox(driver, 'com.tencent.mm:id/b4m') with open(task_file, 'r', encoding="utf8", errors="ignore") as f: for line in f.readlines()[start_id:end_id]: line = line.replace("\n", "").split(",") id, biz = line[0], line[1] print('----------------当前测试序号:' + id + ' 当前测试biz:' + str(biz).lstrip('__biz=')) bizurl = 'https://mp.weixin.qq.com/mp/profile_ext?action=home&__biz=' \ + biz + '&scene=124#wechat_redirect' + '\n' + id # 发送消息 utils.send_msg(driver, bizurl) # 点击链接 utils.click_last_msg_in_talkbox(driver, 'com.tencent.mm:id/nl') # 点击自动关注 add_contact(driver) contacts.append(biz) print('已关注的公众号:' + str(contacts)) # 更新数据库 conn = dbutil.connectdb_wechatcluster() dbutil.insert_by_many_bizinfo_addcontact(conn, contacts) conn.close() # 更新配置文件 start_id = end_id end_id = end_id + 180 # 写入 yaml 文件 update_serial(CONFIG_FILE, udid, start_id, end_id)
def start_authorization_procedure(self): pubkey = self.pubkey.decode('ascii') msg_to_server = self.create_presence_msg(self.client_login, pubkey) LOGGER.info(f'A message has been generated to the server - {msg_to_server}.') try: send_msg(self.connection, msg_to_server) LOGGER.debug(f'Message sent to server.') answer_all = get_msg(self.connection) answer_code = self.answer_server_presence(answer_all) LOGGER.info(f'Received response from server - {answer_code}.\n') self.send_hash_password(answer_all) answer_code = self.answer_server_presence(get_msg(self.connection)) except json.JSONDecodeError: LOGGER.error('Failed to decode received Json string.') self.connection_lack() except IncorrectDataNotDictError: LOGGER.error('Invalid data format received.\n') self.connection_lack() except FieldMissingError as missing_error: LOGGER.error(f'No required field - {missing_error}.\n') self.connection_lack() except IncorrectCodeError as wrong_code: LOGGER.error(f'Invalid code in message - {wrong_code}.') self.connection_lack() except ConnectionResetError: LOGGER.critical('Server connection not established.') self.connection_lack() except ServerError as er: LOGGER.critical(f'{er}') self.is_connected = False self.answer_server.emit(f'{er}', 'login') exit(1) else: LOGGER.info(f'Received response from server - {answer_code}.\n') print(f'Server connection established.') self.progressbar_signal.emit()
def pubkey_request(self, login): """The function of requesting the public key of the client from the server.""" LOGGER.debug(f'Public key request for {login}') request = { ACTION: PUBLIC_KEY_REQUEST, TIME: time.time(), ACCOUNT_NAME: login } with LOCK_SOCKET: try: send_msg(self.connection, request) answer = get_msg(self.connection) except (OSError, json.JSONDecodeError): # self.connection_lost_signal.emit() return if RESPONSE in answer and answer[RESPONSE] == 511: LOGGER.debug(f'Loaded public key for {login}') return answer[DATA] else: LOGGER.error(f'Failed to get the key of the interlocutor {login}. ' f'Answer server {answer}')
def send_group_message(self, group_name, message_text): message = { ACTION: MESSAGE_GROUP, FROM: self.client_login, TO: group_name, TIME: time.time(), MESSAGE_TEXT: message_text } with LOCK_SOCKET: try: send_msg(self.connection, message) answer = get_msg(self.connection) except (ConnectionResetError, ConnectionAbortedError, OSError): LOGGER.critical('Lost server connection.') return False else: if answer[RESPONSE] == 200: LOGGER.info(f'Successfully sent a message for the group {group_name} to the server.') with LOCK_DATABASE: self.database.add_group_message(group_name, self.client_login, message_text) return True
def send_user_message(self, contact_name, message_text): encrypted_message = self.encrypt_decrypt.message_encryption(message_text) message = { ACTION: MESSAGE, FROM: self.client_login, TO: contact_name, TIME: time.time(), MESSAGE_TEXT: encrypted_message } with LOCK_SOCKET: try: send_msg(self.connection, message) answer = get_msg(self.connection) except (ConnectionResetError, ConnectionAbortedError, OSError): LOGGER.critical('Lost server connection.') return False else: if answer[RESPONSE] == 400: LOGGER.info(f'{answer[ERROR]}. User {contact_name} is offline.') return f'User {contact_name} is offline!' LOGGER.debug(f'Message sent: {message},from {self.client_login} username {contact_name}') with LOCK_DATABASE: self.database.save_message(contact_name, 'out', message_text) return True
def start_client_authorization(self, client, message): message_auth = RESPONSE_511 random_str = binascii.hexlify(os.urandom( 64)) # The hexadecimal representation of the binary data # Bytes cannot be in the dictionary, decode (json.dumps -> TypeError) message_auth[DATA] = random_str.decode('ascii') password_hash = self.database.get_hash(message[USER][ACCOUNT_NAME]) hash = hmac.new(password_hash, random_str) server_digest = hash.digest() try: send_msg(client, message_auth) answer = get_msg(client) except OSError: client.close() return client_digest = binascii.a2b_base64(answer[DATA]) if RESPONSE in answer and answer[ RESPONSE] == 511 and hmac.compare_digest( server_digest, client_digest): self.names[message[USER][ACCOUNT_NAME]] = client client_ip, client_port = client.getpeername() try: send_msg(client, RESPONSE_200) except OSError: self.remove_client(message[USER][ACCOUNT_NAME]) self.database.login_user(message[USER][ACCOUNT_NAME], client_ip, client_port, message[USER][PUBLIC_KEY]) LOGGER.info( F'Successful user authentication {message[USER][ACCOUNT_NAME]}' ) self.new_connected_client.emit() else: response = RESPONSE_400 response[ERROR] = 'Wrong password.' try: send_msg(client, response) except OSError: pass self.clients.remove(client) client.close()
def test_send_msg(self): test_socket = TestSocket(self.msg_dict) send_msg(test_socket, self.msg_dict) self.assertEqual(test_socket.encode_json_msg, test_socket.decode_msg)
async def client_msg(self, message, client): LOGGER.debug(f'Parsing a message from a client - {message}') if ACTION in message and TIME in message and USER in message \ and ACCOUNT_NAME in message[USER] \ and message[ACTION] == PRESENCE\ and PUBLIC_KEY in message[USER]: self.checking_new_client(client, message) elif ACTION in message and message[ACTION] == MESSAGE and\ TIME in message and MESSAGE_TEXT in message and TO in message and FROM in message: if message[TO] in self.names: self.messages.append(message) send_msg(client, {RESPONSE: 200}) else: send_msg( client, { RESPONSE: 400, ERROR: 'The user is not registered on the server.' }) elif ACTION in message and message[ACTION] == MESSAGE_GROUP and\ TIME in message and MESSAGE_TEXT in message and TO in message and FROM in message: with LOCK_DATABASE: self.database.add_group_message(message[TO], message[FROM], message[MESSAGE_TEXT]) send_msg(client, {RESPONSE: 200}) self.send_group_message(message) elif ACTION in message and message[ACTION] == GET_CONTACTS and USER in message \ and self.names[message[USER]] == client: answer = { RESPONSE: 202, LIST_INFO: self.database.get_contacts(message[USER]) } send_msg(client, answer) LOGGER.debug( f'Contact list sent to {answer [LIST_INFO]} to user - {message[USER]}\n' ) elif ACTION in message and message[ACTION] == GET_GROUPS and USER in message \ and self.names[message[USER]] == client: answer = { RESPONSE: 202, LIST_INFO: [group[1] for group in self.database.get_groups()] } send_msg(client, answer) LOGGER.debug( f'Groups list sent to {answer [LIST_INFO]} to user - {message[USER]}\n' ) elif ACTION in message and message[ACTION] == GET_MESSAGES_GROUPS and USER in message \ and self.names[message[USER]] == client: answer = { RESPONSE: 202, LIST_INFO: self.database.get_messages_groups() } send_msg(client, answer) LOGGER.debug( f'Groups list sent to {answer [LIST_INFO]} to user - {message[USER]}\n' ) elif ACTION in message and message[ACTION] == ADD_CONTACT \ and ACCOUNT_NAME in message and USER in message \ and self.names[message[USER]] == client: self.database.add_contact(message[USER], message[ACCOUNT_NAME]) send_msg(client, {RESPONSE: 200}) LOGGER.debug( f'New contact added {message[ACCOUNT_NAME]} ay the user {message[USER]}.' ) elif ACTION in message and message[ACTION] == DELETE_CONTACT and ACCOUNT_NAME in message and USER in message \ and self.names[message[USER]] == client: self.database.delete_contact(message[USER], message[ACCOUNT_NAME]) send_msg(client, RESPONSE_200) LOGGER.debug( f'Deleted contact {message [ACCOUNT_NAME]} at the user {message [USER]}' ) elif ACTION in message and message[ACTION] == USERS_REQUEST and ACCOUNT_NAME in message \ and self.names[message[ACCOUNT_NAME]] == client: answer = { RESPONSE: 202, LIST_INFO: [user[0] for user in self.database.users_all()] } send_msg(client, answer) elif ACTION in message and message[ ACTION] == PUBLIC_KEY_REQUEST and ACCOUNT_NAME in message: response = RESPONSE_511 response[DATA] = self.database.get_pubkey(message[ACCOUNT_NAME]) if response[DATA]: try: send_msg(client, response) except OSError: self.remove_client(client) else: response = RESPONSE_400 response[ERROR] = 'There is no public key for this user.' try: send_msg(client, response) except OSError: self.remove_client(client) elif ACTION in message and message[ACTION] == SEND_AVATAR \ and USER in message and ACCOUNT_NAME in message[USER]\ and IMAGE in message[USER]: try: send_msg(client, RESPONSE_200) except OSError: self.remove_client(client) else: img = message[USER][IMAGE] login = message[USER][ACCOUNT_NAME] img_data = base64.b64decode(img) filename = f'img/avatar_{login}.jpg' with open(filename, 'wb') as f: f.write(img_data) self.database.add_image_path(login, filename) LOGGER.debug(f'Added avatar for {client}') elif ACTION in message and message[ACTION] == GET_AVATAR \ and ACCOUNT_NAME in message: login = message[ACCOUNT_NAME] filename = f'img/avatar_{login}.jpg' try: with open(filename, 'rb') as image_file: encoded_img = base64.b64encode( image_file.read()).decode('utf8') except FileNotFoundError: response = RESPONSE_400 response[ERROR] = f'Not found avatar {login}' else: response = RESPONSE_511 response[DATA] = encoded_img try: send_msg(client, response) except OSError: self.remove_client(client) elif ACTION in message and message[ ACTION] == EXIT and ACCOUNT_NAME in message: LOGGER.info(f'User {message [ACCOUNT_NAME]} has disconnected.') user_name = message[ACCOUNT_NAME] self.database.user_logout(user_name) self.clients.remove(self.names[user_name]) self.names[user_name].close() del self.names[user_name] self.disconnected_client.emit() else: msg = {RESPONSE: 400, ERROR: 'Bad Request'} send_msg(client, msg) LOGGER.info(f'Errors sent to client - {msg}.\n')