def screen_shot(self):
screen = self.resultPath + '\img\%s_%s.png' %(self.caseName,self.count)
try:
self.driver.save_screenshot(screen)
except:
L.error('Appium screenshot err, now use adb screenshot')
adb.get_screenshot(self.deviceName,screen)
return screen
def find_element_by_id(self, loc, wait=15):
try:
WebDriverWait(self.driver, wait).until(
lambda driver: driver.find_element_by_id(loc).is_displayed())
return self.driver.find_element_by_id(loc)
except:
L.error(u"%s 页面中未能找到 %s 元素" % (self, loc))
raise Exception('can\'t find id : %s' % loc)
def find_element_by_android_uiautomator(self, loc, wait=15):
try:
WebDriverWait(self.driver, wait).until(
lambda driver: driver.find_element_by_android_uiautomator(
"text(\"%s\")" % loc).is_displayed())
return self.driver.find_element_by_android_uiautomator(
"text(\"%s\")" % loc)
except:
L.error(u"%s 页面中未能找到 %s 元素" % (self, loc))
raise Exception('can\'t find text: %s' % loc)
def requestIPs(self):
# 讯代理
res = requests.get(
'http://www.xdaili.cn/ipagent//freeip/getFreeIps?page=1&rows=10',
timeout=3)
try:
res_dict = json.loads(res.text)
for ip_dict in res_dict['RESULT']['rows']:
address = (ip_dict['ip'] + ':' + ip_dict['port']).encode('u8')
self.ip_pool.append(address) if self.isValidIP(
address) else None
except Exception as e:
# Logging.debug(res.text)
Logging.error(e.message)
def analysis(self, filter):
log_file = self.result_path + '\\log\\%s_%s.log' % (self.logName,
self.count)
error_file = self.result_path + '\status\%s_%s.yaml' % (self.logName,
self.count)
with open(log_file) as f:
for line in f:
if filter in line:
with open(error_file, 'w') as s:
yaml.dump({'error_msg': line}, s)
s.close()
L.error('found log_filter %s!!!' % filter)
f.close()
return error_file
def start_appium(self):
sp = startAppium(self.deviceName)
self.appium_port = sp.main()
count = 10
for i in range(count):
try:
self.driver = self.get_driver(self.appium_port)
# if self.driver:
L.success('appium start % success' % self.deviceName)
return self.driver
#else:
except:
L.error('Faild to start appium %s !' % self.appium_port)
continue
def get_buyer_list(self):
try:
confirmPassengerInitDc = u'https://kyfw.12306.cn/otn/confirmPassenger/initDc'
confirmPassengerUrl = u'https://kyfw.12306.cn/otn/confirmPassenger/getPassengerDTOs'
params = {"_json_att": ""}
res = self.session.post(confirmPassengerInitDc,
params=params,
verify=False)
repeat_token = re.search(
r"var\sglobalRepeatSubmitToken\s=\s'(.*?)'", res.text).group(1)
params = {"_json_att": "", "REPEAT_SUBMIT_TOKEN": repeat_token}
res = self.session.post(confirmPassengerUrl,
params=params,
verify=False)
return json.loads(res.text)['data']['normal_passengers']
except Exception as e:
Logging.error(e)
return None
def __init__(self, username=None, password=None):
self.cookies = cookielib.LWPCookieJar('cookies')
self.session = requests.Session()
self.session.cookies = self.cookies
Logging.debug('<---------- 正在初始化 ---------->')
self.username = username
self.password = password
Logging.debug('<---------- 登录用户名:%s ---------->' %
self.username.encode('u8'))
Logging.debug('<---------- 登录密码:%s ---------->' %
self.password.encode('u8'))
res = self.session.get(self.init_url, verify=False)
if res.status_code == 200:
try:
self.passport_appId = re.search(
r"var\spassport_appId\s=\s'(.*)'", res.text).group(1)
Logging.debug('获取passport_appId:%s' %
self.passport_appId.encode(res.encoding))
self.passport_captcha_url = re.search(
r"var\spassport_captcha\s=\s'(http.*)'", res.text).group(1)
Logging.debug('获取验证码url:%s' %
self.passport_captcha_url.encode(res.encoding))
self.passport_captcha_check_url = re.search(
r"var\spassport_captcha_check\s=\s'(http.*)'",
res.text).group(1)
Logging.debug(
'获取验证码验证url:%s' %
self.passport_captcha_check_url.encode(res.encoding))
self.passport_authuam_url = re.search(
r"var\spassport_authuam\s=\s'(http.*)'", res.text).group(1)
Logging.debug('获取时效验证url:%s' %
self.passport_authuam_url.encode(res.encoding))
self.passport_login = re.search(
r"var\spassport_login\s=\s'(http.*)'", res.text).group(1)
Logging.debug('获取登录url:%s' %
self.passport_login.encode(res.encoding))
self.uamauthclient = "https://kyfw.12306.cn/otn/uamauthclient"
except Exception as e:
Logging.error('%s' % e)
def get_captcha(self):
captcha_param = dict(login_site='E', module='login', rand='sjrand')
Logging.debug('验证码请求参数:%s' % captcha_param)
Logging.debug('<---------- 获取验证码 ---------->')
res = self.session.get(self.passport_captcha_url,
params=captcha_param,
verify=False)
if res.status_code == 200:
captcha_name = 'captcha-image.png'
with open(captcha_name, 'wb') as f:
for data in res.iter_content(chunk_size=1024):
if data:
f.write(data)
f.flush()
f.close()
Logging.debug('<---------- 获取验证码成功 ---------->')
return captcha_name
else:
Logging.error('<---------- 获取验证码失败 ---------->')
return None
def start_test(self):
ini = configIni()
count = ini.get_ini('test_count', 'count')
for i in range(int(count)):
L.info('this is %s test' % str(i + 1))
test_case = get_case().items()
if not test_case:
L.error('testcase not found!!!')
else:
for case_name, case_path in test_case:
L.info('get testcase %s' % case_path)
self.run_case(case_name, case_path, i + 1)
try:
self.driver.quit()
L.success('quit driver %s' % self.appium_port)
except:
L.error('quit driver error %s ' % self.appium_port)
shutdown_appium(self.appium_port, self.deviceName)
analysisLog = analysis_log(case_name, case_path,
self.result_path, i + 1)
analysisLog.get_log_filter()
gen = generatereport(self.result_path)
gen.main()
# -*- coding: utf-8 -*-
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
from Logging import Logging
if __name__ == '__main__':
log = Logging('BBDSpider').get_logging()
log.debug('this is debug message')
log.info('this is info message')
log.error('this is error message')
def login(self):
Logging.debug('<---------- 正在登录 ---------->')
captcha_name = self.get_captcha()
if captcha_name == None or len(captcha_name) == 0:
return False
# self.get_uam()
answer = self.parse_captcha(captcha_name)
captcha_check_param = dict(answer=answer,
login_site='E',
rand='sjrand')
# Logging.debug('验证码请求验证参数:%s' % captcha_check_param)
res = self.session.get(self.passport_captcha_check_url,
params=captcha_check_param,
verify=False)
if res.status_code == 200:
result = json.loads(res.text)
if result['result_code'] == '4':
Logging.debug('<---------- 验证成功 ---------->')
else:
Logging.debug('<---------- %s ---------->' %
result['result_message'])
return False
else:
Logging.error('<---------- 验证失败 ---------->')
return False
login_params = dict(username=self.username,
password=self.password,
appid=self.passport_appId)
res = self.session.post(self.passport_login, params=login_params)
try:
if json.loads(res.text)['result_code'] == 0:
Logging.debug('<---------- 登录成功 ---------->')
Logging.info(res.text)
# Logging.info(requests.utils.dict_from_cookiejar(self.session.cookies))
user_login_res = self.session.post(self.user_login_url,
verify=False,
allow_redirects=False)
Logging.debug(user_login_res.headers)
try:
Logging.debug('<---------- 获取登录重定向url ---------->')
redirect_login_url = user_login_res.headers['Location']
Logging.debug('redirect_login_url: %s' %
redirect_login_url)
Logging.debug(user_login_res.headers)
Logging.debug('<---------- 重定向获取 JSESSIONID ---------->')
redirect_res = self.session.get(redirect_login_url,
verify=False)
Logging.debug(redirect_res.headers)
# Logging.info(requests.utils.dict_from_cookiejar(self.session.cookies))
Logging.debug('<---------- 获取token ---------->')
token = self.get_uam()
if token is None or len(token) == 0:
return False
else:
Logging.debug(
'<---------- 将token保存进cookies ---------->')
res = self.session.post(self.uamauthclient,
params={'tk': token},
verify=False)
res_json = json.loads(res.text)
Logging.debug('<---------- %s ---------->' %
res_json['result_message'].encode('u8'))
if res_json['result_code'] != 0:
return False
self.session.cookies.save(ignore_discard=True)
return True
except Exception as e:
Logging.warning(e)
return False
else:
Logging.debug('<---------- 登录失败 ---------->')
Logging.debug(res.text)
return False
except Exception as e:
Logging.warning(e)
return False
def __init__(self,
start,
to,
trains=None,
dates=None,
seat_types=None,
delay=5):
cookies = cookielib.LWPCookieJar('cookies')
self.session = requests.session()
self.session.cookies = cookies
headers = {
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.101 Safari/537.36',
'Content-Type': 'application/json;charset=UTF-8',
'cache-control': 'no-cache',
'Accept-Encoding': 'gzip, deflate, br'
}
self.session.headers = headers
try:
self.session.cookies.load(ignore_discard=True)
except Exception as e:
Logging.debug(e.message)
Logging.warning(
"cookies 加载失败 !!! 如需订票，请重新运行 'login12306.py'验证登录 !!!")
self.session.cookies = requests.utils.add_dict_to_cookiejar(
self.session.cookies, {
"current_captcha_type":
"Z",
"fp_ver":
"4.5.1",
"RAIL_EXPIRATION":
str(int(round(time.time() * 1000))),
"RAIL_DEVICEID":
"JCTFFU_Ut9rAYiNh49CcBhxCWjxxzfRWpEH9MVv78I-EnTFvtJxZopvJjdKInEm2k0gtSfg06x_xnH2FiRak7_uzab62y0QgnW-GmiRG8GCwa3cxSUeuJeqxG9s_mCn-aQ92yA3h8KElnUty4HAOmm6IYxeiXtm7"
})
self.dates = dates
# 获取站名及其对应缩写
self.country_ls_map, self.country_sl_map, CLeftTicketUrl = self.catch_station_map(
)
if self.country_ls_map is None:
Logging.error('country_ls_map is None')
return
if self.country_sl_map is None:
Logging.error('country_sl_map is None')
return
if CLeftTicketUrl is None:
Logging.error('CLeftTicketUrl is None')
return
self.start_place_ab = self.country_ls_map.get(start)
self.to_place_ab = self.country_ls_map.get(to)
if len(self.start_place_ab) == 0:
Logging.error("param 'start' is invalid")
return
if len(self.to_place_ab) == 0:
Logging.error("param 'to' is invalid")
return
self.target_trains = trains
self.start_place = start
self.to_place = to
self.seat_types = seat_types
self.delay = delay
self.request_ticket_urls, self.save_query_urls = self.config_task_urls(
stringByAppendingPathComponent(DOMAIN, 'otn/' + CLeftTicketUrl))
if len(self.request_ticket_urls) == 0:
Logging.error('request_ticket_urls is None')
return
if len(self.save_query_urls) == 0:
Logging.error('save_query_urls is None')
return
# 检查用户登录状态
self.check_user()
def __init__(self, message):
if (not isinstance(message, "")) or message == "":
self.message = u"帐号密码错误"
else:
self.message = message
Logging.error(self.message)
# Getting Basic Logging Options
logger = Logging().get_logger("scraping")
# Fetching Configuration From Properties File
scraping_script_path = pathlib.Path(__file__).parent.absolute()
handleProperties = HandleProperties()
configuration = handleProperties.read_properties(str(scraping_script_path) + "/Config/Scraping.properties")
# Initializing Variables using Command Line Variables
advertisment_links = set()
print(sys.argv)
total_command_line_arguments = len(sys.argv)
logger.debug("Length of Arguments : " + str(total_command_line_arguments))
if total_command_line_arguments > 5 or total_command_line_arguments < 5:
logger.error("Scraping Module : Incorrect No Of Arguments Passed")
logger.error("Scraping Module : System exiting")
sys.exit()
province_argument = sys.argv[1]
city_argument = sys.argv[2]
type_argument = sys.argv[3]
search_keywords = sys.argv[4]
# Initializing Data From Properties file to Execute Web Scraping
file_path = str(scraping_script_path) + "/Scripts/Location.json"
# Multiple Process Safety Configuration
# Opening File with UTF-8 Encoding
# Both READ-MODE and WRITE-MODE handled using this Method
@lockutils.synchronized('not_thread_process_safe', external=True, fair=True, lock_path=str(scraping_script_path) + "/Lock/")
