def test_report_hourly():
"""
:return:
"""
work_one = log.MyLog("Bill", 'Task1')
assert work_one.report() == "Please pay 50$ to Bill for Task1!"
def test_report_none():
"""
:return:
"""
work_three = log.MyLog("Tony", 'Task4')
assert work_three.report() == "Worker or Task doesnt exists"
def test_report_fixed():
"""
:return:
"""
work_two = log.MyLog("Ted", 'Task3')
assert work_two.report() == "Please pay 500$ to Ted for Task3!"
def info_parse_data(self, parse_data):
data = etree.HTML(parse_data)
li_list = data.xpath('/html/body/div[1]/div[4]/ul/li')
# 循环li_list并存入data_list中,数据量过大可使用迭代器
for li in li_list:
data_dic = {}
data_dic['game_id'] = re.search(r"a/(\d+).htm",
li.xpath('.//a/@href')[0]).group(1)
data_dic['game_name'] = li.xpath('.//div[1]/em/a/text()')[0]
data_dic['logo'] = 'https:' + li.xpath(
'./a/img[@class="gameLogo"]/@lz_src')[0]
data_dic['introduce'] = li.xpath('.//div[1]/p[2]/text()')[0]
data_dic['score'] = li.xpath(
'.//div[1]/div[1]/div/span/text()')[0][:-1]
try:
# 此处两种方法去获取comment数量:1.获取评论json数据，2.解析游戏详情页评论tag信息
# comment_url = 'https://www.3839.com/cdn/comment/view_v2-ac-json-pid-1-fid-'+str(data_dic['game_id'])+'\
# -p-1-order-1-htmlsafe-1-urltype-1-audit-1.htm'
# data_dic['comment_count'] = self.send_comment_request(comment_count)
data_dic['comment_count'] = etree.HTML(self.send_requests(url = 'https:'+ str(li.xpath('.//a/@href')[0]))).\
xpath('/html/body/div[1]/div[3]/div[1]/div[1]/div[2]/a[2]/span/text()')[0]
except Exception as e:
log.MyLog().error(e) #使用logging模块记录发生的错误信息并生成日志文件
data_dic['comment_count'] = ''
data_dic['tag'] = li.xpath('.//div[1]/div[2]/div/span[1]/a/text()')[0]+'、'\
+ li.xpath('.//div[1]/div[2]/div/span[2]/a/text()')[0]+'、'+li.xpath('.//div[1]/div[2]/div/span[3]/a/text()')[0]
data_dic['ranking'] = li.xpath(
'./span[contains(@class,"rank-num")]/text()')[0]
self.info_list.append(data_dic)
def run(self):
print('data_process' + str(threading.currentThread().ident) + '线程开始工作')
while True:
if not self.__url_queue.empty():
comment_dic = self.__url_queue.get(block=False)
self.__url_queue.task_done()
comment_url_list = comment_dic['comment_url_list']
game_id = comment_dic['game_id']
game_name = comment_dic['game_name']
comment_all_data_dic = {}
comment_all_content_list = []
for comment_url in comment_url_list:
comment_content_dic = {}
n = 3 # 设置三次请求json数据的次数，若三次出错，则使用自定义log模块记录错误信息
while n > 0:
try:
response = requests.get(comment_url,
headers=self.headers,
timeout=3).json()
except Exception as e:
log.MyLog().error(e) #使用自定义模块捕捉异常
n -= 1
else:
content_list = response['content']
for comment_content in content_list:
comment_content_dic[
'comment_id'] = comment_content['id']
comment_content_dic[
'user_id'] = comment_content['uid']
comment_content_dic[
'username'] = comment_content['username']
comment_content_dic[
'portrait'] = 'https://' + comment_content[
'avatar'][4:]
comment_content_dic[
'creat_time'] = comment_content['time']
comment_content_dic[
'content'] = comment_content['comment']
try:
comment_content_dic['reply_count'] = len(
comment_content['reply'])
except Exception as e:
comment_content_dic['reply_count'] = 0
else:
comment_content_dic['reply_count'] = len(
comment_content['reply'])
comment_all_content_list.append(
comment_content_dic)
break
comment_all_data_dic['game_id'] = game_id
comment_all_data_dic['game_name'] = game_name
comment_all_data_dic[
'game_comment_list'] = comment_all_content_list
self.__data_queue.put(comment_all_data_dic, block=False)
else:
print('data_process' + str(threading.currentThread().ident) +
'线程工作完成')
break
def getresult():
ret = os.system(
"/home/ubuntu/tpcc-mysql/tpcc_start -h 127.0.0.1 -P 3306 -dtpcc1000 -u root -pxidian320 -w 20 -c 32 -r 300 -l 300 ->/home/ubuntu/tpcc-mysql/tpcc-output-log"
)
time = []
stime.sleep(150)
if ret == 0:
try:
f = open('/home/ubuntu/tpcc-mysql/tpcc-output-log', 'r')
except Exception, e:
log.MyLog(str(e))