def run(self): try: urls_article = [ 'https://mp.weixin.qq.com/s?src=11×tamp=1541559601&ver=1229&signature=ixTsG-RvK8H58t6D-CpW6olWI8hA52Wz-FRb12ZcrNG-lxR20YutoyLYUr-RB3w8WHjE1petjDcbbxZVxTChvPWM27qszWu0Z3zonjx8SEQB5mmgm1O9Eu*5qsFhnBCH&new=1' ] entity = None backpack_list = [] ftp_list = [] ftp_info = None for page_count, url in enumerate(urls_article): # if page_count < 15: # continue html = requests.get(url) # 确定account信息 name = pq(html.text)('#js_name').text() account_name = pq( html.text)('.profile_meta_value').eq(0).text() log('---{}---{}---'.format(name, account_name)) account = Account() account.name = name account.account = account_name account.get_account_id() article = Article() try: article.create(url, account) except RuntimeError as run_error: log('找不到浏览器 {}'.format(run_error)) log('第{}条 文章标题: {}'.format(page_count, article.title)) log("当前文章url: {}".format(url)) entity = JsonEntity(article, account) log('当前文章ID: {}'.format(entity.id)) # if entity.id in ids: # log('当前文章已存在,跳过') # continue backpack = Backpack() backpack.create(entity) backpack_list.append(backpack.create_backpack()) # self.save_to_mysql(entity) self.save_to_mongo(entity.to_dict()) # ftp包 ftp_info = Ftp(entity) name_xml = ftp_info.hash_md5(ftp_info.url) log('当前文章xml: {}'.format(name_xml)) self.create_xml(ftp_info.ftp_dict(), name_xml) ftp_list.append(name_xml) # if page_count >= 3: # break log("发包") # todo 发包超时,修改MTU if ftp_info is not None: entity.uploads_ftp(ftp_info, ftp_list) if entity: # entity.uploads(backpack_list) entity.uploads_datacenter_relay(backpack_list) entity.uploads_datacenter_unity(backpack_list) log("发包完成") except Exception as e: log("解析公众号错误 {}".format(e)) if 'chrome not reachable' in str(e): raise RuntimeError('chrome not reachable')
def run(self): # self.set_name() # while True: account_list = ['有看投',] entity = None backpack_list = [] for name in account_list: self.name = name html_account = self.account_homepage() if html_account: html, account_of_homepage = html_account else: continue log('start 公众号: ', self.name) urls_article = self.urls_article(html) account = Account() account.name = self.name account.account = account_of_homepage account.get_account_id() # account.account_id = 126774646 for page_count, url in enumerate(urls_article): # if page_count < 35: # continue article = Article() article.create(url, account) log('文章标题:', article.title) log("第{}条".format(page_count)) entity = JsonEntity(article, account) backpack = Backpack() backpack.create(entity) backpack_list.append(backpack.create_backpack()) import pymongo conn = pymongo.MongoClient('mongo') # 上传数据库 sql = ''' INSERT INTO account_http(article_url, addon, account, account_id, author, id, title) VALUES (%s, %s, %s, %s, %s, %s, %s) ''' _tuple = ( article.url, datetime.datetime.now(), entity.account, entity.account_id, entity.author, entity.id, entity.title ) uploads_mysql(config_mysql, sql, _tuple) if page_count == 4: break log("发包") if entity: # entity.uploads(backpack_list) # entity.uploads_datacenter_relay(backpack_list) entity.uploads_datacenter_unity(backpack_list) print('end')
def run(self): count = 0 while True: # ADD_COLLECTION 补采账号 get_account 日常采集; 使用account_list 兼容单个账号和账号列表 account_list = ADD_COLLECTION if ADD_COLLECTION else self.get_account( ) # length = len(threading.enumerate()) # 枚举返回个列表 # log.info('当前运行的线程数为:{}'.format(threading.active_count())) count += 1 log.info('第{}次'.format(count)) if account_list is None: log.info('调度队列为空,休眠5秒') time.sleep(5) continue for account_name in account_list: try: self.search_name = account_name html_account = self.account_homepage() if html_account: html = html_account else: log.info('{}|找到不到微信号'.format(account_name)) continue urls_article = self.urls_article(html) # 确定account信息 account = Account() account.name = self.name account.account = account_name account.tags = self.get_tags() account.get_account_id() # 判重 ids = self.dedup(account_name) if JUDEG else '' entity = None backpack_list = [] ftp_list = [] ftp_info = None for page_count, url in enumerate(urls_article): # if page_count < 15: # continue article = Article() try: article.create(url, account) except RuntimeError as run_error: log.info('微信验证码错误 {}'.format(run_error)) log.info('第{}条 文章标题: {}'.format( page_count, article.title)) log.info("当前文章url: {}".format(url)) entity = JsonEntity(article, account) log.info('当前文章ID: {}'.format(entity.id)) if entity.id in ids and JUDEG is True: log.info('当前文章已存在,跳过') # if page_count >= 20: # log.info('超过20篇文章,跳出') # break continue backpack = Backpack() backpack.create(entity) backpack_list.append(backpack.create_backpack()) # self.save_to_mysql(entity) # self.save_to_mongo(entity.to_dict()) # ftp包 ftp_info = Ftp(entity) name_xml = ftp_info.hash_md5(ftp_info.url) log.info('当前文章xml: {}'.format(name_xml)) self.create_xml(ftp_info.ftp_dict(), name_xml) ftp_list.append(name_xml) # break log.info("开始发包") # todo 发包超时,修改MTU if ftp_info is not None: entity.uploads_ftp(ftp_info, ftp_list) log.info("ftp发包完成") if entity and backpack_list: # entity.uploads(backpack_list) entity.uploads_datacenter_relay(backpack_list) entity.uploads_datacenter_unity(backpack_list) log.info("数据中心,三合一,发包完成") except Exception as e: log.exception("解析公众号错误 {}".format(e)) if 'chrome not reachable' in str(e): raise RuntimeError('chrome not reachable') if ADD_COLLECTION: break
def run(self): count = 0 while True: # ADD_COLLECTION 补采账号 get_account 日常采集; 使用account_list 兼容单个账号和账号列表 account_list = ADD_COLLECTION if ADD_COLLECTION else self.get_account( ) # length = len(threading.enumerate()) # 枚举返回个列表 log.info('当前运行的线程数为:{}'.format(threading.active_count())) log.info('当前运行的进程:{}'.format( multiprocessing.current_process().name)) count += 1 log.info('第{}次'.format(count)) if account_list is None: log.info('调度队列为空,休眠5秒') time.sleep(5) continue for account_name in account_list: try: self.search_name = account_name html_account = self.account_homepage() if html_account: html = html_account else: log.info('{}|找到不到微信号'.format(account_name)) continue urls_article = self.urls_article(html) # 确定account信息 account = Account() account.name = self.name account.account = account_name account.tags = self.get_tags() account.get_account_id() # 判重 查底层 # ids = self.dedup(account_name) if JUDEG else '' # 判重 redis sentenced_keys = account.account + ' ' + str( account.account_id) keys = hash_md5(sentenced_keys) log.info('keys: {}'.format(keys)) dedup_result = self.dedup_redis(keys) post_dedup_urls = [] entity = None backpack_list = [] ftp_list = [] ftp_info = None for page_count, url in enumerate(urls_article): try: # if page_count > 5: # break article = Article() article.create(url, account, self.proxies) log.info('第{}条 文章标题: {}'.format( page_count, article.title)) log.info("当前文章url: {}".format(url)) entity = JsonEntity(article, account) log.info('当前文章ID: {}'.format(entity.id)) article_date = datetime.datetime.fromtimestamp( int(str(article.time)[:-3])) day_diff = datetime.date.today( ) - article_date.date() if day_diff.days > 15: log.info( '超过采集interval最大15天 的文章不采集,已采集{}条文章'.format( page_count)) self.count_articles(page_count) break if dedup_result: # title_time_str = entity.title + str(entity.time) # title_time_md5 = hash_md5(title_time_str) if entity.id in dedup_result: log.info('当前文章已存在,跳过') continue else: post_dedup_urls.append(entity.id) else: # title_time_str = entity.title + str(entity.time) # title_time_md5 = hash_md5(title_time_str) post_dedup_urls.append(entity.id) # dedup_result = self.dedup_redis(entity) # if dedup_result: # log.info('当前文章已存在,跳过') # ids = ids.append({'key': entity.id, 'urls': entity.url}) # if entity.id in ids and JUDEG is True: # log.info('当前文章已存在,跳过') # continue backpack = Backpack() backpack.create(entity) backpack_list.append(backpack.create_backpack()) # self.save_to_mysql(entity) # self.save_to_mongo(entity.to_dict()) # ftp包 ftp_info = Ftp(entity) name_xml = ftp_info.hash_md5(ftp_info.url) log.info('当前文章xml: {}'.format(name_xml)) self.create_xml(ftp_info.ftp_dict(), name_xml) ftp_list.append(name_xml) except Exception as run_error: log.info('微信解析文章错误 {}'.format(run_error)) continue log.info("开始发包") if entity and backpack_list: # 直接发底层 # entity.uploads(backpack_list) entity.uploads_datacenter_relay(backpack_list) entity.uploads_datacenter_unity(backpack_list) log.info("数据中心,三合一,发包完成") else: log.info('包列表为空,不发送数据') continue # todo 发包超时,修改MTU if ftp_info is not None: entity.uploads_ftp(ftp_info, ftp_list) log.info("ftp发包完成") if post_dedup_urls: log.info('上传判重中心key:{} urls:{}'.format( keys, post_dedup_urls)) url = 'http://47.100.53.87:8008/Schedule/CacheWx' data = [{ "key": keys, "sourceNodes": "1", "sourceType": "2", "urls": post_dedup_urls }] r = requests.post(url, data=json.dumps(data), timeout=self.timeout) log.info('上传判重中心结果{}'.format(r.status_code)) except Exception as e: log.exception("解析公众号错误 {}".format(e)) time.sleep(30) if ('chrome not reachable' in str(e)) or ('Message: timeout' in str(e)): raise RuntimeError('chrome not reachable') if ADD_COLLECTION: break
def run(self): count = 0 while True: count += 1 log.info('第{}次'.format(count)) account_list = ADD_COLLECTION if ADD_COLLECTION else self.account_list( ) # if account_list: # continue # for account_name in account_list: try: for account_name in account_list: log.info('第{}次'.format(count)) self.search_name = account_name html_account = self.account_homepage() if html_account: html = html_account else: log.info('找到不到微信号首页: '.format(account_name)) continue urls_article = self.urls_article(html) # 确定account信息 account = Account() account.name = self.name account.account = account_name account.get_account_id() # 判重 ids = self.dedup(account_name) entity = None backpack_list = [] ftp_list = [] ftp_info = None for page_count, url in enumerate(urls_article): # if page_count < 15: # continue article = Article() try: article.create(url, account) except RuntimeError as run_error: log.info('找不到浏览器 {}'.format(run_error)) log.info('第{}条 文章标题: {}'.format( page_count, article.title)) log.info("当前文章url: {}".format(url)) entity = JsonEntity(article, account) log.info('当前文章ID: {}'.format(entity.id)) if entity.id in ids: log.info('当前文章已存在,跳过') continue backpack = Backpack() backpack.create(entity) backpack_list.append(backpack.create_backpack()) # self.save_to_mysql(entity) self.save_to_mongo(entity.to_dict()) # ftp包 ftp_info = Ftp(entity) name_xml = ftp_info.hash_md5(ftp_info.url) log.info('当前文章xml: {}'.format(name_xml)) self.create_xml(ftp_info.ftp_dict(), name_xml) ftp_list.append(name_xml) # if page_count >= 3: # break log.info("发包") # todo 发包超时,修改MTU if ftp_info is not None: entity.uploads_ftp(ftp_info, ftp_list) if entity: # entity.uploads(backpack_list) entity.uploads_datacenter_relay(backpack_list) entity.uploads_datacenter_unity(backpack_list) log.info("发包完成") except Exception as e: log.exception("解析公众号错误 {}".format(e)) if 'chrome not reachable' in str(e): raise RuntimeError('chrome not reachable') continue