def parse_conf_class(conf_class):
'''
解析配置文件的所有属性，然后设置env模块的所有属性
'''
if sys.getdefaultencoding() != 'utf-8':
reload(sys)
sys.setdefaultencoding('utf-8')
new_conf = []
for var in dir(conf_class):
if (not var.startswith("__")) and (not var.endswith("__")):
new_conf.append(var)
def_conf = []
for var in dir(env):
if (not var.startswith("__")) and (not var.endswith("__")):
def_conf.append(var)
for conf in new_conf:
if conf in def_conf:
setattr(env, conf, getattr(conf_class, conf)) #设置值
else:
log.step_warning("conf [%s] not found in module env." % conf)
# if result path not set in conf, set where the "run.py" file locates.
if env.RESULT_PATH == "":
env.RESULT_PATH = os.path.dirname(
os.path.abspath(
inspect.stack()[-1]
[1])) #获取当前模块最开始调用路径 '[0][1]上层模块调用路径' '[1][1当前模块路径'
def __wait_for_disappearing(cls):
t = 0
while t < 120:
t = t + 1
try:
elements = env.threadlocal.BROWSER.find_elements(cls.by, cls.value)
except NoSuchElementException:
log.step_normal("Element [%s]: NoSuchElementException." % cls.__name__)
elements = []
except BadStatusLine:
log.step_warning("Element [%s]: BadStatusLine." % cls.__name__)
continue
except UnexpectedAlertPresentException:
log.step_warning("Element [%s]: UnexpectedAlertPresentException." % cls.__name__)
if len(elements) == 0:
return True
else:
time.sleep(0.5)
log.step_normal("Element [%s]: WairForDisappearing... Found [%s] Element. Tried [%s] Times." % (cls.__name__, len(elements), t))
return False
def __wait(cls):
t = 0
while t < 300:
t = t + 1
try:
elements = env.threadlocal.BROWSER.find_elements(cls.by, cls.value)
except NoSuchElementException:
log.step_normal("Element [%s]: NoSuchElementException." % cls.__name__)
elements = []
except BadStatusLine:
log.step_warning("Element [%s]: BadStatusLine." % cls.__name__)
except UnexpectedAlertPresentException:
log.step_warning("Element [%s]: UnexpectedAlertPresentException." % cls.__name__)
if len(elements) == 0:
time.sleep(0.5)
log.step_normal("Element [%s]: Wait 0.5 second, By [%s :: %s :: %s]" % (cls.__name__, cls.by, cls.value, cls.index))
else:
if len(elements) > 1:
log.step_normal("Element [%s]: There are [%s] Elements!" % (cls.__name__, len(elements)))
break
if len(elements) < cls.index + 1:
log.step_fail("Element [%s]: Element Index Issue! There are [%s] Elements! Index=[%s]" % (cls.__name__, len(elements), cls.index))
def AlertSendKeys(cls, value):
log.step_normal("AlertSendKeys [%s]" % value)
try:
env.threadlocal.BROWSER.switch_to.alert.send_keys(value)
env.threadlocal.BROWSER.switch_to.default_content()
except:
log.step_warning(str(sys.exc_info()))
def parse_conf_class(conf_class):
"""
Parse the configure class, get all the properties of the class, then set
their corresponding value in module "env".
AUTHOR
Henry.Wang, 2015-02-11
"""
if sys.getdefaultencoding() != "utf-8":
reload(sys)
sys.setdefaultencoding("utf-8")
new_conf = []
for var in dir(conf_class):
if (not var.startswith("__")) and (not var.endswith("__")):
new_conf.append(var)
def_conf = []
for var in dir(env):
if (not var.startswith("__")) and (not var.endswith("__")):
def_conf.append(var)
for conf in new_conf:
if conf in def_conf:
setattr(env, conf, getattr(conf_class, conf))
else:
log.step_warning("conf [%s] not found in module env." % conf)
# if result path not set in conf, set where the "run.py" file locates.
if env.RESULT_PATH == "":
env.RESULT_PATH = os.path.dirname(os.path.abspath(inspect.stack()[-1][1]))
def parse_conf_class(conf_class):
'''
Parse the configure class, get all the properties of the class, then set
their corresponding value in module "env".
AUTHOR
Henry.Wang, 2015-02-11
'''
if sys.getdefaultencoding() != 'utf-8':
reload(sys)
sys.setdefaultencoding('utf-8')
new_conf = []
for var in dir(conf_class):
if (not var.startswith("__")) and (not var.endswith("__")):
new_conf.append(var)
def_conf = []
for var in dir(env):
if (not var.startswith("__")) and (not var.endswith("__")):
def_conf.append(var)
for conf in new_conf:
if conf in def_conf:
setattr(env, conf, getattr(conf_class, conf))
else:
log.step_warning("conf [%s] not found in module env." % conf)
# if result path not set in conf, set where the "run.py" file locates.
if env.RESULT_PATH == "":
env.RESULT_PATH = os.path.dirname(
os.path.abspath(inspect.stack()[-1][1]))
def copy(src, dst):
try:
if os.path.isdir(src):
shutil.copytree(src, dst)
else:
shutil.copy(src, dst)
except:
log.step_warning(str(sys.exc_info()))
def copy(src, dst):
try:
if os.path.isdir(src):
shutil.copytree(src, dst) #拷贝目录
else:
shutil.copy(src, dst) #拷贝文件
except:
log.step_warning(str(sys.exc_info()))
def SwitchToDefaultContent(cls):
log.step_normal("SwitchToDefaultContent()")
try:
# env.threadlocal.BROWSER.switch_to_default_content()
env.threadlocal.BROWSER.switch_to.default_content()
except:
log.step_warning("env.threadlocal.BROWSER.switch_to.default_content()")
pass
def SwitchToDefaultWindow(cls):
log.step_normal("SwitchToDefaultWindow()")
log.step_normal("Switch To The Default Window of: %s" % str(env.threadlocal.BROWSER.window_handles))
try:
env.threadlocal.BROWSER.switch_to.window(env.threadlocal.BROWSER.window_handles[0])
except:
log.step_warning("env.threadlocal.BROWSER.switch_to.window(env.threadlocal.BROWSER.window_handles[0])")
pass
def remove_readonly(fn, path, excinfo):
if os.path.exists(path):
try:
if fn is os.rmdir:
os.chmod(path, stat.S_IWRITE)
os.rmdir(path) #删除目录
elif fn is os.remove:
os.chmod(path, stat.S_IWRITE)
os.remove(path) #删除文件
except:
log.step_warning(str(sys.exc_info()))
def remove_readonly(fn, path, excinfo):
if os.path.exists(path):
try:
if fn is os.rmdir:
os.chmod(path, stat.S_IWRITE)
os.rmdir(path)
elif fn is os.remove:
os.chmod(path, stat.S_IWRITE)
os.remove(path)
except:
log.step_warning(str(sys.exc_info()))
def praserJsonFile():
if len(env.p.lrange('proxy_list', 0, -1)) == 0:
log.step_normal('proxy_list 为空！')
try:
data = urllib2.urlopen(env.PROXY).read()
s = json.loads(data)
if len(s['msg']) == 0:
proxy = Check(random.choice(list(set(
s['data']['proxy_list']))))
return proxy
else:
time.sleep(30)
return praserJsonFile()
except Exception, e:
log.step_warning('打开代理地址失败:[%s]' % e)
time.sleep(30)
return praserJsonFile()
def generate_html_brush(test_cases=[], countdown=True):
folders = common.get_sub_folder_names(
os.path.join(env.RESULT_PATH, "result"))
reports = []
for folder in folders:
if len(folder) == 36:
reports.append(folder)
with open(os.path.join(env.RESULT_PATH, "result", "brush.html"), "w") as f:
f.write(html_source_header())
f.write(html_source_body_brush(countdown=countdown))
f.write(html_source_table1_brush())
i = 0
car = {}
for report in sorted(reports, reverse=True): #倒序排序遍历
BRAND = common.get_value_from_conf(
os.path.join(env.RESULT_PATH, "result", report, "status.ini"),
"BRAND")
CLICKBRAND = common.get_value_from_conf(
os.path.join(env.RESULT_PATH, "result", report, "status.ini"),
"CLICKBRAND")
BROWSE = common.get_value_from_conf(
os.path.join(env.RESULT_PATH, "result", report, "status.ini"),
"BROWSE")
if BRAND is not None and BRAND not in car:
car[BRAND] = [1, int(CLICKBRAND), int(BROWSE)]
elif BRAND is not None:
car[BRAND][0] += 1
car[BRAND][1] += int(CLICKBRAND)
car[BRAND][2] += int(int(BROWSE))
else:
log.step_warning('该代理ip没有打开百度搜索 [%s]' % env.BRAND)
for key, value in car.items():
f.write(
html_source_table2_brush((key, value[0], value[1], value[2])))
f.write(html_source_version_info())
f.write(html_source_foot())
def insert_db(ipdate):
lock = threading.Lock()
try:
# 获取一个数据库连接，注意如果是UTF-8类型的，需要制定数据库
lock.acquire()
conn = env.conn
cur = conn.cursor() # 获取一个游标对象
# 插入数据
ISOTIMEFORMAT = '%Y-%m-%d %X'
ipdate.append(time.strftime(ISOTIMEFORMAT, time.localtime()))
log.step_normal('ipdate:[%s]' % ipdate)
cur.execute(
"INSERT INTO ip_log(ip,address,keyword,url,click,error,page,rank,created_at) VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s)",
ipdate)
cur.close() # 关闭游标
conn.commit() # 向数据库中提交任何未解决的事务，对不支持事务的数据库不进行任何操作
# conn.close() # 关闭到数据库的连接，释放数据库资源
lock.release()
log.step_normal('数据沉淀完成！')
except (MySQLdb.OperationalError, Exception) as e:
log.step_warning('数据库操作异常:[%s]' % e)
def get_local_ip(ip):
try:
proxy_support = urllib2.ProxyHandler({'http': 'http://' + ip})
opener = urllib2.build_opener(proxy_support, urllib2.HTTPHandler)
urllib2.install_opener(opener)
request = urllib2.Request('http://ip.chinaz.com/getip.aspx')
# request.add_header("cookie", env.COOKIE)
request.add_header("User-Agent", getUA())
fp = urllib2.urlopen(request)
mybytes = fp.read()
# note that Python3 does not read the html code as string
# but as html code bytearray, convert to string with
mystr = mybytes.decode('utf-8')
fp.close()
ip = mystr.find("ip")
add = mystr.find("address")
ip = mystr[ip + 4:add - 2]
address = mystr[add + 9:-2]
return [ip, address]
except (HTTPError, URLError, Exception) as e:
log.step_warning('获取ip地址失败---> %s' % e)
return [ip, 'address']
def mkdirs(dir_path):
try:
if not os.path.exists(dir_path):
os.makedirs(dir_path)
except:
log.step_warning(str(sys.exc_info()))
def quit_browser(browser):
try:
browser.quit()
except:
log.step_warning(str(sys.exc_info()))
def launch_browser(url):
'''
Launch a new browser, and set the parameters for the browser.
'''
if env.threadlocal.TESTING_BROWSER.upper() == 'FIREFOX':
fp = FirefoxProfile()
fp.native_events_enabled = False
if env.FIREFOX_BINARY == '':
try:
env.THREAD_LOCK.acquire()
browser = webdriver.Firefox(firefox_profile=fp)
except:
if isinstance(env.RESERVED_FIREFOX_BINARY,
str) and env.RESERVED_FIREFOX_BINARY != "":
browser = webdriver.Firefox(
firefox_profile=fp,
firefox_binary=FirefoxBinary(
firefox_path=env.RESERVED_FIREFOX_BINARY))
else:
try:
log.step_warning("try to start firefox again!")
time.sleep(20)
browser = webdriver.Firefox(firefox_profile=fp)
except:
log.handle_error()
return False
finally:
env.THREAD_LOCK.release()
else:
browser = webdriver.Firefox(
firefox_profile=fp,
firefox_binary=FirefoxBinary(firefox_path=env.FIREFOX_BINARY))
elif env.threadlocal.TESTING_BROWSER.upper() == 'CHROME':
if env.DRIVER_OF_CHROME == '':
print('DRIVER_OF_CHROME is empty.')
return False
os.environ['webdriver.chrome.driver'] = env.DRIVER_OF_CHROME
browser = webdriver.Chrome(executable_path=env.DRIVER_OF_CHROME)
elif env.threadlocal.TESTING_BROWSER.upper() == 'IE':
if env.DRIVER_OF_IE == '':
print('DRIVER_OF_IE is empty.')
return False
# os.popen('TASKKILL /F /IM iexplore.exe')
os.popen('TASKKILL /F /IM IEDriverServer.exe')
dc = DesiredCapabilities.INTERNETEXPLORER.copy()
dc['nativeEvents'] = False
dc['acceptSslCerts'] = True
os.environ['webdriver.ie.driver'] = env.DRIVER_OF_IE
browser = webdriver.Ie(executable_path=env.DRIVER_OF_IE,
capabilities=dc)
elif env.threadlocal.TESTING_BROWSER.upper() == 'PHANTOMJS':
browser = webdriver.PhantomJS(
r'D:\\Emma\\qiqi\\pop\\drivers\\phantomjs\\phantomjs.exe')
if not env.BROWSER_VERSION_INFO.has_key(env.threadlocal.TESTING_BROWSER):
env.BROWSER_VERSION_INFO[
env.threadlocal.TESTING_BROWSER] = browser.capabilities['version']
browser.set_window_size(1920, 1080)
browser.set_window_position(0, 0)
browser.set_page_load_timeout(300)
browser.implicitly_wait(0)
browser.get(url)
return browser
def launch_browser(url):
'''
Launch a new browser, and set the parameters for the browser.
'''
if env.threadlocal.TESTING_BROWSER.upper() == 'FIREFOX':
fp = FirefoxProfile()
fp.native_events_enabled = False
if env.FIREFOX_BINARY == '':
try:
env.THREAD_LOCK.acquire()
browser = webdriver.Firefox(firefox_profile=fp)
except:
if isinstance(env.RESERVED_FIREFOX_BINARY, str) and env.RESERVED_FIREFOX_BINARY != "":
browser = webdriver.Firefox(firefox_profile=fp,
firefox_binary=FirefoxBinary(firefox_path=env.RESERVED_FIREFOX_BINARY))
else:
try:
log.step_warning("try to start firefox again!")
time.sleep(20)
browser = webdriver.Firefox(firefox_profile=fp)
except:
log.handle_error()
return False
finally:
env.THREAD_LOCK.release()
else:
browser = webdriver.Firefox(firefox_profile=fp,
firefox_binary=FirefoxBinary(firefox_path=env.FIREFOX_BINARY))
elif env.threadlocal.TESTING_BROWSER.upper() == 'CHROME':
if env.DRIVER_OF_CHROME == '':
print ('DRIVER_OF_CHROME is empty.')
return False
os.environ['webdriver.chrome.driver'] = env.DRIVER_OF_CHROME
browser = webdriver.Chrome(executable_path=env.DRIVER_OF_CHROME)
elif env.threadlocal.TESTING_BROWSER.upper() == 'IE':
if env.DRIVER_OF_IE == '':
print ('DRIVER_OF_IE is empty.')
return False
# os.popen('TASKKILL /F /IM iexplore.exe')
os.popen('TASKKILL /F /IM IEDriverServer.exe')
dc = DesiredCapabilities.INTERNETEXPLORER.copy()
dc['nativeEvents'] = False
dc['acceptSslCerts'] = True
os.environ['webdriver.ie.driver'] = env.DRIVER_OF_IE
browser = webdriver.Ie(executable_path=env.DRIVER_OF_IE,
capabilities=dc)
elif env.threadlocal.TESTING_BROWSER.upper() == 'PHANTOMJS':
browser = webdriver.PhantomJS(r'E:\\AutomationWork\\phantomjs-1.9.8-windows\\phantomjs.exe')
if not env.BROWSER_VERSION_INFO.has_key(env.threadlocal.TESTING_BROWSER):
env.BROWSER_VERSION_INFO[env.threadlocal.TESTING_BROWSER] = browser.capabilities['version']
browser.set_window_size(1366, 758)
browser.set_window_position(0, 0)
browser.set_page_load_timeout(300)
browser.implicitly_wait(0)
browser.get(url)
return browser
def quit_browser(browser):
try:
browser.quit()
except:
log.step_warning(str(sys.exc_info()))
def delete_folder(folder_path):
try:
shutil.rmtree(folder_path) #递归删除
except:
log.step_warning(str(sys.exc_info()))
def delete_folder(folder_path):
try:
shutil.rmtree(folder_path)
except:
log.step_warning(str(sys.exc_info()))
def mkdirs(dir_path):
try:
if not os.path.exists(dir_path): #判断是否存在这个路径
os.makedirs(dir_path)
except:
log.step_warning(str(sys.exc_info()))