def refresh_fixtures(): # Get fixtures and update in DB logger.info("Refreshing fixtures...") api_call_stat = list(Stat.query("api-call"))[0] api_stat_dict = api_call_stat.stat_value.as_dict() logger.info(api_stat_dict) logger.info("Current API call count : {}".format(api_stat_dict["count"])) if api_stat_dict["count"] < int(config.get("MaxApiCall")): fixtures_list = api.get_new_fixtures(config.get("TeamId")) for f in fixtures_list: if f["statusShort"] == "NS" or f["statusShort"] == "TBD": Fixture(fixture_id=f["fixture_id"], home=f["homeTeam"]["team_name"], away=f["awayTeam"]["team_name"], start_time=int(f["event_timestamp"]), status=f["statusShort"], result=None, league=f["league_id"]).save() api_stat_dict["count"] += 1 api_call_stat.stat_value = api_stat_dict api_call_stat.save() else: logger.error("API call limit exceeded. Fixtures not refreshed")
def crawl_group(self): """爬小组成员页""" group_name = config.get('group', 'id') log.info(group_name) group = GroupList() if group.total_members is 0 or group.total_pages is 0: # 小组成员页获取错误 raise Exception('所爬小组:{}\t 总人数:{}\t 总页数{}\n大概率账号被ban'.format( group_name, group.total_members, group.total_pages)) else: log.info('所爬小组:{}\t 总人数:{}\t 总页数{}'.format(group_name, group.total_members, group.total_pages)) # 设置开始爬的位置,倒着爬 start_page = (lambda page_num: group.total_pages if page_num is -1 else page_num)(int( config.get('group', 'start_page'))) end_page = (lambda page_num: 0 if page_num is -1 else page_num)(int( config.get('group', 'end_page'))) # 开始爬,每倒爬step页换一个headers和proxies step = int(config.get('group', 'skip_page')) for page_range in range(start_page, end_page, step): # 换一个headers和proxies group = GroupList() try: self.crawl_group_members(page_range, page_range + step, group) except Exception as e: raise Exception('crawl_group: {}'.format(e))
class TestBid(object): """投标测试用例""" @pytest.mark.parametrize('case', bid_data) def test_bid(self, case, init_web): driver = init_web login_page = LoginPage(driver) home_page = HomePage(driver) bid_page = BidPage(driver) # 登录 login_page.login(config.get('data', 'phone'), config.get('data', 'password')) # 断言 try: # 确认首页已经打开 home_page.opened() # 首页,点击投标,进入投标详情页 home_page.get_bid_button().click() # 投标详情页,定位可用余额输入框 element_available_balance = bid_page.get_available_balance() element_available_balance.send_keys(case['amount']) # 投标详情页,定位投标按钮 element_bid_button = bid_page.get_bid_button() # 断言 assert case['expected'] == getattr(element_bid_button, 'text') print('用例执行成功') except Exception as e: # 截图 home_page.screen_shot() print('用例执行失败') raise e
def valid_proxy(self, headers=''): """ 验证代理IP是否有效 :param headers: 请求头 :type headers: dict :return: 是否有效 :rtype: bool """ # FIXME: 似乎代理IP不起作用? url = config.get('proxy', 'test_url') proxies = {'http': 'http://' + self.ip_port} if headers is '': headers = { config.get('headers', 'user_agent_info'): config.get_ua() } try: r = requests.get(url, headers=headers, allow_redirects=False, proxies=proxies, timeout=20) if r.status_code == 200: self.is_valid = True else: self.is_valid = False raise Exception(r.text) except Exception as e: raise Exception('{} valid_proxy: {}'.format(self.ip_port, e)) finally: return self.is_valid
def __init__(self): """初始化方法,获取配置文件内容""" self.log_name = config.get("log", "log_name") self.log_level = config.get("log", "log_level") self.sh_level = config.get("log", "sh_level") self.fh_level = config.get("log", "fh_level") # 拼接日志文件路径 self.file_path = os.path.join(LOG_DIR, config.get("log", "file_name"))
def get_new_fixtures(team_id): url = config["FootballApiBaseUrl"] + "/fixtures/team/" + team_id headers = { 'X-RapidAPI-Host': config.get("FootballApiHost"), "X-RapidAPI-Key": config.get("FootballApiKey") } r = requests.get(url, headers=headers) rdict = json.loads(r.text) return rdict["api"]["fixtures"]
def __init__(self): # 连接数据库 self.coon = pymysql.connect( host=config.get('mysql', 'host'), # 数据库地址 port=config.getint('mysql', 'port'), # 端口 user=config.get('mysql', 'user'), # 账号 password=config.get('mysql', 'password'), # 密码 database=config.get('mysql', 'database'), # 数据库名 charset="utf8") # 创建一个游标 self.cur = self.coon.cursor()
def set_headers(self): try: config.use_section('headers') self.headers = { config.get(option='connection_info'): config.get(option='connection'), config.get(option='user_agent_info'): config.get_ua() } log.info('{}'.format(self.headers)) except Exception as e: raise Exception('set_headers: {}'.format(e))
def get_mobile(): """ 随机生成手机号 :return: """ suffix = config.get('db', 'table_name') + config.get('db', 'db_name') prefix = [ '158', '136', '130', '131', '132', '133', '149', '134', '150', '152', '153' ] middle = random.randint(10000, 99999) mobile = random.choice(prefix) + str(middle) + suffix return mobile
def get_result(fixture_id): print(config.get("FootballApiBaseUrl")) headers = { 'X-RapidAPI-Host': config.get("FootballApiHost"), "X-RapidAPI-Key": config.get("FootballApiKey") } url = config["FootballApiBaseUrl"] + "/fixtures/id/" + str(fixture_id) r = requests.get(url, headers=headers) rdict = json.loads(r.text) fixture_info = rdict["api"]["fixtures"][0] resp = {} if fixture_info["statusShort"] == "FT": resp['score'] = fixture_info["score"] resp['events'] = fixture_info['events'] return resp
def mylogger(name): #新建日志收集器 logger = logging.getLogger(name) #设置日志的输出级别 logger.setLevel('DEBUG') # fmt = ('%(asctime)s-%(name)s-%(levelname)s-日志信息:%(message)s- [%(filename)s:%(lineno)d ]') # formatter = logging.Formatter(fmt=fmt) formatter = logging.Formatter(config.get('logger', 'fmt')) #新建日志输出器 #指定日志输出到控制台 sh = logging.StreamHandler() sh.setLevel('DEBUG') sh.setFormatter(formatter) #指定日志输出到文件中 fh = logging.FileHandler(constants.log_file + '/case.log', encoding='utf-8', mode='w') fh.setLevel('DEBUG') fh.setFormatter(formatter) #将日志输出器和日志输出器配合使用 logger.addHandler(sh) logger.addHandler(fh) return logger
def crawl_contacts(self): """爬小组用户的关注列表页并写入数据库中对应用户名的表中""" try: self.login() # 关注用户表,表名对应当前用户id user_id = config.get('user', 'id') contacts = ContactsTable(table_name=user_id) c = ContactsList(self.s) total_members, total_pages = c.total_members, c.total_pages log.info('共关注了{}位用户,列表页共有{}页'.format(c.total_members, c.total_pages)) # for page_num in range(1, total_pages + 1): log.info('当前进度:[{}/{}]'.format(page_num, total_pages)) try: page_members = c.get_contacts_from_page(page_num) except Exception as e: raise Exception('因为{}, 无法爬取页:{} 至 {}'.format( e, page_num, total_pages)) else: contacts.insert(page_members) # 防止IP或者账号被ban的睡眠 time.sleep(random.randint(3, 20)) except Exception as e: raise Exception('crawl_contacts: {}'.format(e))
def __init__(self): # 把这些参数放到配置文件里面去做,然后在这里读取配置文件 host = config.get("mysql", "host") user = config.get("mysql", "user") password = config.get("mysql", "password") port = int(config.get("mysql", "port")) # 创建连接 logger.info("连接sql") self.mysql = pymysql.Connect(host=host, user=user, password=password, port=port, charset='utf8') # 创建游标,设置返回字典 self.cursor = self.mysql.cursor( pymysql.cursors.DictCursor) #创建此类浮标,查询结果返回的是一个字典
def print_group_table(self): """打印拉取到的小组成员页数据""" members = MembersTable(table_name=config.get('group', 'id')) items = members.fetch_all() num_existed = len(items) log.info('当前共有: {}'.format(num_existed)) log.info('随机打印一个用户的信息') members.fetch_one_basic_infos().print_basic_infos()
def update_sheets(fixture): logger.info( "Updating Google Sheets after {} vs {} match, Fixture: {}".format( fixture.home, fixture.away, fixture.fixture_id)) spreadsheet_id = config.get("SpreadSheetId") service = build_sheets_service() users = get_users_from_db() first_row = [ "Last Updated after {} vs {}".format(fixture.home, fixture.away) ] actual_result_str = "{}-{}, {}, {}".format(str(fixture.result.home_goals), str(fixture.result.away_goals), fixture.result.scorers, str(fixture.result.first_event)) second_row = ["Actual Result -->", actual_result_str] empty_row = [] header_row = [ "User", "Total Points", "Latest Prediction", "Latest Prediction Points" ] sheet_values = [first_row, second_row, empty_row, empty_row, header_row] fixture_dict = {} for u in users: curr_result = u.curr_prediction.result if u.curr_prediction.fixture not in fixture_dict: fixture_dict[u.curr_prediction.fixture] = get_fixture( u.curr_prediction.fixture) user_fixture = fixture_dict[u.curr_prediction.fixture] curr_prediction_str = "{} vs {} : {}-{}, {}, {}".format( user_fixture.home, user_fixture.away, str(curr_result.home_goals), str(curr_result.away_goals), curr_result.scorers, str(curr_result.first_event)) row = [ u.user_id, u.total_points, curr_prediction_str, u.curr_prediction.points ] sheet_values.append(row) body = {'values': sheet_values} result = service.spreadsheets().values().update( spreadsheetId=spreadsheet_id, range="A1", valueInputOption="RAW", body=body).execute() print('{0} cells updated.'.format(result.get('updatedCells'))) # if __name__ == "__main__": # r = Result(home_goals=2, away_goals=3, scorers=["A","B"], first_event = 33) # f = Fixture(fixture_id=219, home="Tottenham Hotspur", away="Manchester United", start_time=1560928951, status="NS", result=r, league=2) # update_sheets(f)
def check_api_limit(): api_call_stat = list(Stat.query("api-call"))[0] api_stat_dict = api_call_stat.stat_value.as_dict() logger.info(api_stat_dict) logger.info("Current API call count : {}".format(api_stat_dict["count"])) api_stat_dict["count"] += 1 api_call_stat.stat_value = api_stat_dict api_call_stat.save() return api_stat_dict["count"] < int(config.get("MaxApiCall"))
def __init__(self, session=requests.Session(), group_id='', headers=None, proxies=None): """ 从小组拉取成员信息,小组成员页总页数 """ super(GroupList, self).__init__() # 设置默认的参数 self.s = session if group_id is '': group_id = config.get('group', 'id') if headers is None: self.set_headers() else: self.headers = headers if proxies is None: self.set_proxies() else: self.proxies = proxies # 获取小组总人数 self.group_members_url = "https://www.douban.com/group/{}/members".format( group_id) try: # 建立连接 r = self.s.get(self.group_members_url, headers=self.headers, proxies=self.proxies, allow_redirects=False, timeout=20) if r.status_code != 200: # 连接失败 raise Exception('status: {}, url: {}'.format( r.status_code, r.url)) except Exception as e: raise Exception('GroupList: {}'.format(e)) else: url_group_buffer = r.text # IMPROVE: 需要在BeautifulSoup和re中择一进行解析 soup = BeautifulSoup(url_group_buffer, features="lxml") span_count = soup.find('span', {'class': 'count'}) total_members = 0 # 获取小组成员总数 if span_count: total_members_str = re.search(r'\(共(\d*)人\)', span_count.text) if total_members_str: total_members = int(total_members_str.group(1)) self.total_members = total_members # 获取小组成员总页数 paginator_div = soup.find('div', {'class': 'paginator'}) total_pages = 0 if paginator_div: total_pages_str = paginator_div.findAll('a')[-2].string if total_pages_str: total_pages = int(total_pages_str) self.total_pages = total_pages
class RechargeTestCase(unittest.TestCase): """充值和取现接口的验证""" excel = DoExcel(os.path.join(DATA_DIR, "cases.xlsx"), "recharge&withdraw") all_cases = excel.read_data() url = config.get("api", "url") @classmethod def setUpClass(cls): cls.session = HttpSession() cls.mysql = DoMysql() @classmethod def tearDownClass(cls): cls.session.session_close() cls.mysql.close() @data(*all_cases) def test_recharge(self, case): log.info(f"验证接口:{case.interface},接口url:{case.url}") # 第一步:准备测试数据,替换动态参数 # 替换配置文件中的固定参数 case.data = sub_conf(case.data) # 替换动态生成的参数 if "*phone*" in case.data: # 注册时,手机号码为动态注册的号码 phone = random_phone() case.data = case.data.replace("*phone*", phone) if "*amount*" in case.data: # 充值金额不足时,需要先获取库中的值 sql = f"select LeaveAmount from member where MobilePhone = '{eval(case.data)['mobilephone']}'" money = float(self.mysql.find_one(sql)[0]) case.data = case.data.replace("*amount*", str(money + 1000)) # 未登录时取现/充值,没有返回码 if case.expected_code: expected_code = str(case.expected_code) else: expected_code = None # 数据库校验 if case.check_sql: case.check_sql = sub_conf(case.check_sql) old_money = float(self.mysql.find_one(case.check_sql)[0]) # 第二步:发送请求到接口,获取实际结果 res_code = self.session.http_session(method=case.method, url=self.url + case.url, data=eval( case.data)).json()["code"] # 处理上下接口关联问题 if case.interface == "注册": # 注册后,将注册的账号和密码设置为缓存数据类的属性,以便其他用例读取 setattr(CacheData, "phone", phone) setattr(CacheData, "pwd", "123qwe")
class MyLogging(object): log_name = config.get("log", "log_name") log_level = config.get("log", "log_level") sh_level = config.get("log", "sh_level") fh_level = config.get("log", "fh_level") # 拼接日志文件路径 file_path = os.path.join(LOG_DIR, TIME + "-" + config.get("log", "file_name")) def __new__(cls, *args, **kwargs): """创建对象""" # 第一步:创建日志收集器对象,并设置等级 log = logging.getLogger(cls.log_name) log.setLevel(cls.log_level) # 第二步:创建日志输出渠道,并设置等级 # 1、输出到控制台 sh = logging.StreamHandler() sh.setLevel(cls.sh_level) # 2、输出到日志文件 fh = logging.FileHandler(filename=cls.file_path, encoding="utf8") fh.setLevel(cls.fh_level) # 第三步:添加日志输出渠道到日志收集器中 log.addHandler(sh) log.addHandler(fh) # 第四步:指定日志输出格式 # 创建日志输出格式对象 fot = logging.Formatter( "[%(asctime)s][%(filename)s-->line:%(lineno)d][%(levelname)s] %(message)s" ) # 将日志输出格式对象绑定到日志输出渠道中 sh.setFormatter(fot) fh.setFormatter(fot) # # 第五步:移除日志 # log.removeHandler(sh) # log.removeHandler(fh) return log
def sub_conf(data): """替换动态参数为配置文件中的内容""" # 通过search方法进行匹配,能匹配到数据才进行数据处理 while re.search(r"#(.+?)#", data): res = re.search(r"#(.+?)#", data) try: # 通过提取的字段,去配置文件中读取对应的数据内容 value = config.get("data", res.group(1)) except: value = getattr(CacheData, res.group(1)) data = re.sub(res.group(), value, data) return data
def print_group_match_me(self): """随机打印一个打印从成员表中过滤得到的用户""" members = MembersTable(table_name=config.get('database', 'table_name_members')) items = members.fecth_all_match_me() print('----------------------------------------------------------------------------------') print('过滤得到的用户数: ', len(items)) for item in items: member = members.conv_member(item) member.print_infos() # member.print_basic_infos() print() break
def get_logger(name): logger = logging.getLogger(name) # 名为case_log的日志收集器 logger.setLevel(config.get('log_info', 'collect_level')) # 设定收集的级别 #指定格式 formatter = logging.Formatter(config.get('log_info', 'log_famtter')) # 新建指定的输出渠道: # 指定输出渠道 handler console_handler = logging.StreamHandler() # 指定输出到console控制台 console_handler.setLevel(config.get('log_info', 'output_level')) # 设定输出信息的级别 console_handler.setFormatter(formatter) # 指定输出文本渠道 handler curTime = time.strftime("%Y-%m-%d %H%M", time.localtime()) file_handler = logging.FileHandler(contants.log_dir + "/Web_Autotest_{0}.log".format(curTime), encoding='utf-8') file_handler.setLevel(config.get('log_info', 'output_level')) # 设定输出信息的级别 file_handler.setFormatter(formatter) # 配合关系 logger.addHandler(console_handler) logger.addHandler(file_handler) # 收集日志 return logger
class BidloanTestCase(unittest.TestCase): """投资竞标接口的验证""" excel = DoExcel(os.path.join(DATA_DIR, "cases.xlsx"), "bidloan") all_cases = excel.read_data() url = config.get("api", "url") @classmethod def setUpClass(cls): cls.http = HttpSession() cls.mysql = DoMysql() @classmethod def tearDownClass(cls): cls.http.session_close() cls.mysql.close() @data(*all_cases) def test_bidloan(self, case): log.info(f"验证接口:{case.interface},接口url:{case.url}") # 第一步:准备用例数据 # 替换配置文件中的固定参数 case.data = sub_conf(case.data) # 替换动态生成的参数 if "*phone*" in case.data: # 注册时,手机号码为动态注册的号码 phone = random_phone() case.data = case.data.replace("*phone*", phone) if "*memberId*" in case.data: # 想要用户id不存在,先找到最大用户id,加1即可 member_id = self.mysql.find_one( "select max(id) as memberId from member")[0] case.data = case.data.replace("*memberId*", str(member_id + 1)) if "*loanId*" in case.data: # 想要项目id不存在,先找到最大项目id,加1即可 loan_id = self.mysql.find_one( "select max(id) as loanId from loan")[0] case.data = case.data.replace("*loanId*", str(loan_id + 1)) if "*amount*" in case.data: # 想要可投金额不足,先找到可投金额的最大值,加大值即可 sql = f"select Amount from loan where Id = {eval(case.data)['loanId']}" case.data = case.data.replace( "*amount*", str(self.mysql.find_one(sql)[0] + 1000)) # 未登录时投资竞标,没有返回码 if case.expected_code: case.expected_code = str(case.expected_code) else:
def data_replace(data): """替换动态参数""" while re.search(r'#(\w+)#', data): res = re.search(r'#(\w+)#', data) # 提取要替换的内容 r_data = res.group() # 获取要替换的字段 key = res.group(1) try: # 从配置文件读取字段 value = config.get('data', key) except Exception: # 从ConText类中读取 value = getattr(ConText, key) # 替换 data = re.sub(r_data, str(value), data) return data
def crawl_group_member(self): """ 爬取小组成员首页数据,更新插入数据库表members中,并将更新后的用户从小组对应的表中删除 """ try: # 登录 self.login() # 用户表:存储已经更新完数据,具有完整首页数据字段 users = MembersTable() # 小组成员表:数据是待更新首页信息的小组成员:只包含基本信息 group_id = config.get('group', 'id') group = MembersTable(table_name=group_id) # 从数据库拉取没有对应首页字段内容的用户,迭代爬取首页内容,更新字段 while True: # 从小组表中随机拉取一个成员 user = group.fetch_one_not_updated() if users.is_existed_by_id(user.usr_id): # 如果当前ID已经存在,则继续操作 print('用户:{} 已存在'.format(user.usr_id)) continue # 小组成员首页数据获取 # IMPROVE: 每次self.s.get()的时候重新设置新的头部和代理 user.set_cur_member_page(self.s) # 更新数据信息:注册日期 user.set_register_date(user.get_register_date()) # 更新数据信息:共同喜好 user.set_common_likes(user.get_common_likes()) # 更新数据信息:关注的小组数目,关注的用户数,被关注的用户数 user.set_group_num(user.get_group_num()) user.set_contacts_num(user.get_contacts_num()) user.set_rev_contacts_num(user.get_rev_contacts_num()) # 更新数据信息:书影音的数据 user.set_stats_book(user.get_stats_book()) user.set_stats_movie(user.get_stats_movie()) user.set_stats_music(user.get_stats_music()) # 打印爬到的成员信息 # user.print_infos() # 将更新完信息的user插入用户表中 users.insert([user]) # 从小组成员表中删除该成员user group.delete([user]) # 防止IP或者账号被ban的睡眠 time.sleep(random.randint(15, 20)) except Exception as e: raise Exception('crawl_home_page: {}'.format(e))
def replace(data): p = "#(.*?)#" # 正则表达式 while re.search(p, data): print(f'请求数据:{data}') m = re.search(p, data) # 从任意位置开始找,找第一个就返回Match object, 如果没有找None g = m.group(1) # 拿到参数化的KEY try: v = config.get('data', g) # 根据KEY取配置文件里面的值 except configparser.NoOptionError as e: # 如果配置文件里面没有的时候,去Context里面取 if hasattr(Context, g): v = getattr(Context, g) else: print('找不到参数化的值') raise e print(f'V 的值:{v}') # 替换后的内容,继续用data接受 data = re.sub(p, v, data, count=1) # 查找替换,count查找替换的次数 return data
def http_request(self, method, url, data=None, json=None, cookie=None): if type(data) == str: data = ast.literal_eval(data) # 将str转成字典 # 拼接URL url = config.get('api', 'pre_url') + url logger.debug('请求url:{0}'.format(url)) logger.debug('请求data:{0}'.format(data)) if method.upper() == 'GET': try: res = self.session.request(method=method, url=url, params=data, cookies=cookie) except Exception as e: res = 'Error:GET请求报错:{0}'.format(e) elif method.upper() == 'POST': if json: try: res = self.session.request(method=method, url=url, json=json, cookies=cookie) except Exception as e: res = 'Error:POST请求报错:{0}'.format(e) else: try: res = self.session.request(method=method, url=url, data=data, cookies=cookie) except Exception as e: res = 'Error:POST请求报错:{0}'.format(e) else: print('你的请求方式不正确!') res = 'Error:请求方法不对报错:%s' % method # res = 'Error:请求方法不对报错:{0}'.format(method) return res
class AddTestCase(unittest.TestCase): """加标接口的验证""" excel = DoExcel(os.path.join(DATA_DIR, "cases.xlsx"), "add") all_cases = excel.read_data() url = config.get("api", "url") @classmethod def setUpClass(cls): cls.http = HttpSession() cls.mysql = DoMysql() @classmethod def tearDownClass(cls): cls.http.session_close() cls.mysql.close() @data(*all_cases) def test_add(self, case): log.info(f"验证接口:{case.interface},接口url:{case.url}") # 第一步:准备用例数据 # 替换配置文件中的固定参数 case.data = sub_conf(case.data) # 替换动态生成的参数 if "*phone*" in case.data: # 注册时,手机号码为动态注册的号码 phone = random_phone() case.data = case.data.replace("*phone*", phone) if "*memberId*" in case.data: # 想要用户id不存在,先找到最大用户id,加1即可 member_id = self.mysql.find_one( "select max(id) as memberId from member")[0] case.data = case.data.replace("*memberId*", str(member_id + 1)) # 未登录时新增项目,没有返回码 if case.expected_code: case.expected_code = str(case.expected_code) else:
def Wshttps(self, url, data, method): logger.info("实例化一个client对象") # 拼接URL self.url = config.get('api', 'pre_url') + url self.method = method self.data = eval(data) logger.info('请求URL:{0}'.format(self.url)) logger.info('请求METHOD:{0}'.format(self.method)) logger.info('请求DATA:{0}'.format(self.data)) client = Client(self.url) try: logger.info("发送一个webservice请求") resp = eval("client.service.{0}({1})".format( self.method, self.data)) msg = resp.retInfo logger.info("返回信息:{}".format(resp.retInfo)) logger.info("返回码:{}".format(resp.retCode)) except suds.WebFault as e: logger.info(e.fault.faultstring) msg = e.fault.faultstring return msg
def __init__(self, db_addr='', table_name=config.get('database', 'table_name_proxy')): super(ProxyTable, self).__init__(db_addr, table_name) self.create()