def test_report_hourly(): """ :return: """ work_one = log.MyLog("Bill", 'Task1') assert work_one.report() == "Please pay 50$ to Bill for Task1!"
def test_report_none(): """ :return: """ work_three = log.MyLog("Tony", 'Task4') assert work_three.report() == "Worker or Task doesnt exists"
def test_report_fixed(): """ :return: """ work_two = log.MyLog("Ted", 'Task3') assert work_two.report() == "Please pay 500$ to Ted for Task3!"
def info_parse_data(self, parse_data): data = etree.HTML(parse_data) li_list = data.xpath('/html/body/div[1]/div[4]/ul/li') # 循环li_list并存入data_list中,数据量过大可使用迭代器 for li in li_list: data_dic = {} data_dic['game_id'] = re.search(r"a/(\d+).htm", li.xpath('.//a/@href')[0]).group(1) data_dic['game_name'] = li.xpath('.//div[1]/em/a/text()')[0] data_dic['logo'] = 'https:' + li.xpath( './a/img[@class="gameLogo"]/@lz_src')[0] data_dic['introduce'] = li.xpath('.//div[1]/p[2]/text()')[0] data_dic['score'] = li.xpath( './/div[1]/div[1]/div/span/text()')[0][:-1] try: # 此处两种方法去获取comment数量:1.获取评论json数据,2.解析游戏详情页评论tag信息 # comment_url = 'https://www.3839.com/cdn/comment/view_v2-ac-json-pid-1-fid-'+str(data_dic['game_id'])+'\ # -p-1-order-1-htmlsafe-1-urltype-1-audit-1.htm' # data_dic['comment_count'] = self.send_comment_request(comment_count) data_dic['comment_count'] = etree.HTML(self.send_requests(url = 'https:'+ str(li.xpath('.//a/@href')[0]))).\ xpath('/html/body/div[1]/div[3]/div[1]/div[1]/div[2]/a[2]/span/text()')[0] except Exception as e: log.MyLog().error(e) #使用logging模块记录发生的错误信息并生成日志文件 data_dic['comment_count'] = '' data_dic['tag'] = li.xpath('.//div[1]/div[2]/div/span[1]/a/text()')[0]+'、'\ + li.xpath('.//div[1]/div[2]/div/span[2]/a/text()')[0]+'、'+li.xpath('.//div[1]/div[2]/div/span[3]/a/text()')[0] data_dic['ranking'] = li.xpath( './span[contains(@class,"rank-num")]/text()')[0] self.info_list.append(data_dic)
def run(self): print('data_process' + str(threading.currentThread().ident) + '线程开始工作') while True: if not self.__url_queue.empty(): comment_dic = self.__url_queue.get(block=False) self.__url_queue.task_done() comment_url_list = comment_dic['comment_url_list'] game_id = comment_dic['game_id'] game_name = comment_dic['game_name'] comment_all_data_dic = {} comment_all_content_list = [] for comment_url in comment_url_list: comment_content_dic = {} n = 3 # 设置三次请求json数据的次数,若三次出错,则使用自定义log模块记录错误信息 while n > 0: try: response = requests.get(comment_url, headers=self.headers, timeout=3).json() except Exception as e: log.MyLog().error(e) #使用自定义模块捕捉异常 n -= 1 else: content_list = response['content'] for comment_content in content_list: comment_content_dic[ 'comment_id'] = comment_content['id'] comment_content_dic[ 'user_id'] = comment_content['uid'] comment_content_dic[ 'username'] = comment_content['username'] comment_content_dic[ 'portrait'] = 'https://' + comment_content[ 'avatar'][4:] comment_content_dic[ 'creat_time'] = comment_content['time'] comment_content_dic[ 'content'] = comment_content['comment'] try: comment_content_dic['reply_count'] = len( comment_content['reply']) except Exception as e: comment_content_dic['reply_count'] = 0 else: comment_content_dic['reply_count'] = len( comment_content['reply']) comment_all_content_list.append( comment_content_dic) break comment_all_data_dic['game_id'] = game_id comment_all_data_dic['game_name'] = game_name comment_all_data_dic[ 'game_comment_list'] = comment_all_content_list self.__data_queue.put(comment_all_data_dic, block=False) else: print('data_process' + str(threading.currentThread().ident) + '线程工作完成') break
def getresult(): ret = os.system( "/home/ubuntu/tpcc-mysql/tpcc_start -h 127.0.0.1 -P 3306 -dtpcc1000 -u root -pxidian320 -w 20 -c 32 -r 300 -l 300 ->/home/ubuntu/tpcc-mysql/tpcc-output-log" ) time = [] stime.sleep(150) if ret == 0: try: f = open('/home/ubuntu/tpcc-mysql/tpcc-output-log', 'r') except Exception, e: log.MyLog(str(e))