def start_xvfb(module):
try:
xvfb = Xvfb(width=1280, height=720)
xvfb.start()
atexit.register(xvfb.stop)
except:
module.fail_json(msg="xvfb broke")
class XvfbRobot(object):
"""
A robot library for creating virtual display on demand
"""
ROBOT_LIBRARY_SCOPE = "GLOBAL"
ROBOT_LIBRARY_VERSION = VERSION
_display = None
def start_virtual_display(self, width=1440, height=900, colordepth=24, **kwargs):
"""Starts virtual display which will be
destroyed after test execution will be end
*Arguments:*
- width: a width to be set in pixels
- height: a height to be set in pixels
- color_depth: a color depth to be used
- kwargs: extra parameters
*Example:*
| Start Virtual Display |
| Start Virtual Display | 1920 | 1080 |
| Start Virtual Display | ${1920} | ${1080} | ${16} |
"""
if self._display is None:
logger.info("Using virtual display: '{0}x{1}x{2}'".format(width, height, colordepth))
self._display = Xvfb(int(width), int(height), int(colordepth), **kwargs)
self._display.start()
atexit.register(self._display.stop)
class BaseTestCase(unittest.TestCase):
def setUp(self):
self.xvfb = os.environ.get("ENABLE_XVFB", False)
self.browser = os.environ.get("BROWSER", "Chrome")
if self.xvfb:
self.vdisplay = Xvfb(width=1280, height=720)
self.vdisplay.start()
if self.browser == "Firefox":
self.driver = self.get_ff_driver()
else:
self.driver = self.get_chrome_driver()
self.load = self.driver.get
def tearDown(self):
if self.driver:
self.driver.quit()
if self.xvfb and self.vdisplay:
self.vdisplay.stop()
def get_ff_driver(self):
return webdriver.Firefox()
def get_chrome_driver(self):
opts = Options()
if "TRAVIS" in os.environ: # github.com/travis-ci/travis-ci/issues/938
opts.add_argument("--no-sandbox")
# Fix for https://code.google.com/p/chromedriver/issues/detail?id=799
opts.add_experimental_option("excludeSwitches",
["ignore-certificate-errors"])
return webdriver.Chrome(chrome_options=opts)
def get_mtgox_info(input_vars):
vdisplay = Xvfb()
vdisplay.start()
driver = webdriver.Firefox()
driver.get("http://www.mtgox.com")
uelem = driver.find_element_by_name("username")
pelem = driver.find_element_by_name("password")
lelem = driver.find_element_by_name("LOGIN")
uelem.send_keys(input_vars['mtgoxId'])
time.sleep(0.25)
pelem.send_keys(input_vars['mtgoxPassword'])
time.sleep(0.25)
lelem.click()
time.sleep(0.25)
driver.get("https://www.mtgox.com/trade/funding-options")
time.sleep(0.25)
logout_button = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CLASS_NAME, "logout")))
address_elem = driver.find_elements_by_xpath("/html/body/div[2]/div[2]/div[3]/div[3]/section/div/div[4]/div[2]/div/p/strong")
print address_elem
new_address=address_elem.text
def main():
import sys
import imageio
import pandas as pd
import matplotlib.pyplot as plt
from xvfbwrapper import Xvfb
plt.ioff()
# with Xvfb() as xvfb:
# plt.ioff()
vdisplay = Xvfb()
vdisplay.start()
if len(sys.argv)<5:
subsampleRate = 10
else:
subsampleRate = int(sys.argv[4])
if len(sys.argv)<6:
speedup = 10
else:
speedup = int(sys.argv[5])
bounds2video(sys.argv[1],sys.argv[2],sys.argv[3],subsampleRate,speedup)
vdisplay.stop()
def __init__(self):
log.start(logfile=time.strftime("log/%Y%m%d%H%M%S")+".log",logstdout=False)
log.msg("initiating crawler...",level=log.INFO)
self.crawler_id = self.get_crawler_id()
log.msg("crawler id is %s" % self.crawler_id,level=log.INFO)
self.r.set('crawler:ip:%s' % self.crawler_id,utils.get_external_ip())
self.r.set('crawler:port:%s' % self.crawler_id,settings.REDIS_LOCAL_PORT)
self.r.set('crawler:mapping_port:%s' % self.crawler_id,settings.REDIS_LOCAL_MAPPING_PORT)
log.msg("crawler ip is %s, port is %d" % (utils.get_external_ip(),settings.REDIS_LOCAL_PORT),level=log.INFO)
account = self.get_account()
self.username = account[0]
self.password = account[1]
log.msg("crawler account got",level=log.INFO)
self.r_local.set('crawler:status:%s' % self.crawler_id, 'good')
self.r_local.set('crawler:update_time:%s' % self.crawler_id, datetime.datetime.utcnow().strftime("%s"))
log.msg("local crawler status set",level=log.INFO)
heartbeat_thread = threading.Thread(target=self.maintain_local_heartbeat)
heartbeat_thread.start()
log.msg("local crawler heartbeat started",level=log.INFO)
if platform.system() == "Linux":
#on linux, use virtual display
vdisplay = Xvfb()
vdisplay.start()
co = ChromeOptions()
#TODO: Disable image after log in
#TODO: optimize memory usage
co.add_experimental_option("prefs",{"profile.default_content_settings":{"popups":1}})
#co.add_experimental_option("prefs",{"profile.default_content_settings":{"popups":1,"images":2,"media":2}})
self.driver = webdriver.Chrome(chrome_options=co)
self.driver.set_window_size(640,960)
class MainTest(unittest.TestCase):
def setUp(self):
self.xvfb = Xvfb(width=1280, height=720)
self.xvfb.start()
self.browser = webdriver.Firefox()
self.addCleanup(self.browser.quit)
def test_main(self):
self.browser.get("http://www.youtube.com")
#self.browser.get("http://www.youtube.com//results?search_query=ionic")
search = self.browser.find_element_by_id("masthead-search-term")
search.send_keys('angular',Keys.RETURN)
self.browser.implicitly_wait(2)
self.browser.save_screenshot("youtube.png")
'''
with open('youtube.html','w') as html:
page = driver.page_source
html.write(page.encode('utf-8'))
'''
link_elements = self.browser.find_elements_by_xpath("//a[@title]")
f = csv.writer(open("youtube.csv", "w"))
f.writerow(["Name", " Link"])
for element in link_elements:
if len(element.text) > 5:
name = element.text
f.writerow([name.encode('utf-8')," "+ element.get_attribute("href")])
def test_start(self):
xvfb = Xvfb()
self.addCleanup(xvfb.stop)
xvfb.start()
display_var = ':{}'.format(xvfb.new_display)
self.assertEqual(display_var, os.environ['DISPLAY'])
self.assertIsNotNone(xvfb.proc)
class Xvfb(Plugin):
def options(self, parser, env):
super(Xvfb, self).options(parser, env)
parser.add_option("--with-xvfb-options", action="store",
dest="xvfb_options",
default=env.get("NOSE_WITH_XVFB_OPTIONS"),
help="Options to pass to Xvfb. Comma delimited with "
"equals as separators if necessary. "
"E.g. \"extension=SELINUX, once\". Currently, there is no "
"way to provide options that begin with a +. This is a limitation "
"in xvfbwrapper. Repetition is not allowed. [NOSE_WITH_XVFB_OPTIONS]")
def configure(self, options, noseconfig):
super(Xvfb, self).configure(options, noseconfig)
self.xvfb_options = {}
if options.xvfb_options:
opts = [x.strip() for x in options.xvfb_options.split(",")]
for item in opts:
key, sign, value = item.partition("=")
if not value:
value = ''
self.xvfb_options[key] = value
def begin(self):
logger.info('Starting xvfb virtual display 1024x768 with %s' % self.xvfb_options)
self.vdisplay = XvfbWrapper(width=1024, height=768, **self.xvfb_options)
self.vdisplay.start()
def finalize(self, result):
logger.info('Stopping xvfb virtual display')
self.vdisplay.stop()
def start_xvfb():
from xvfbwrapper import Xvfb
if "DISPLAY" in os.environ:
del os.environ["DISPLAY"]
xvfb = Xvfb()
xvfb.start()
atexit.register(xvfb.stop)
def test_start_with_arbitrary_kwargs(self):
xvfb = Xvfb(nolisten='tcp', noreset=None)
self.addCleanup(xvfb.stop)
xvfb.start()
display_var = ':{}'.format(xvfb.new_display)
self.assertEqual(display_var, os.environ['DISPLAY'])
self.assertIsNotNone(xvfb.proc)
class TestJavascript(unittest.TestCase):
def setUp(self):
# Configure a web service.
handler = http.server.SimpleHTTPRequestHandler
address = ("127.0.0.1", 9999)
socketserver.TCPServer.allow_reuse_address = True # Prevents address conflicts.
httpd = socketserver.TCPServer(address, handler)
# Start the web service in a separate thread as deamon.
httpd_thread = threading.Thread(target=httpd.serve_forever)
httpd_thread.setDaemon(True)
httpd_thread.start()
# Start a display.
self.display = Xvfb()
self.display.start()
# Start the browser driver for selenium testing.
self.driver = webdriver.Firefox()
self.driver.get("http://localhost:9999/index.html")
def test_javascript(self):
# Create an instance of the selenium Firefox driver.
error = self.driver.find_elements_by_id("error")[0].text
self.assertEqual(error, "")
def tearDown(self):
self.display.stop()
class Webdriver(unittest.TestCase):
def setUp(self):
self.vdisplay = Xvfb(width=1280, height=720)
self.vdisplay.start()
# if(not vdisplay.start()):
# fo = open(LOG_FILE, "a")
# fo.write("Xvfbfailure||"+str(TREATMENTID)+"||"+str(ID)+"\n")
# fo.close()
# sys.exit(0)
if(BROWSER=='firefox'):
if (platform.system()=='Darwin'):
self.driver = webdriver.Firefox()
elif (platform.system()=='Linux'):
self.driver = webdriver.Firefox(proxy=proxy)
else:
print "Unidentified Platform"
sys.exit(0)
elif(BROWSER=='chrome'):
if (platform.system()=='Darwin'):
chromedriver = "./experiment/chromedriver/chromedriver_mac"
os.environ["webdriver.chrome.driver"] = chromedriver
self.driver = webdriver.Chrome(executable_path=chromedriver)
elif (platform.system() == 'Linux'):
chromedriver = "./experiment/chromedriver/chromedriver_linux"
os.environ["webdriver.chrome.driver"] = chromedriver
chrome_option = webdriver.ChromeOptions()
chrome_option.add_argument("--proxy-server=yogi.pdl.cmu.edu:3128" )
self.driver = webdriver.Chrome(executable_path=chromedriver, chrome_options=chrome_option)
else:
print "Unidentified Platform"
sys.exit(0)
else:
print "Unsupported Browser"
sys.exit(0)
self.driver.implicitly_wait(10)
self.base_url = "https://www.google.com/"
self.verificationErrors = []
self.driver.set_page_load_timeout(40)
self.accept_next_alert = True
def test_webdriver(self):
fo = open(AD_FILE, "w")
fo.close()
driver = self.driver
driver.get(SITE)
count = 0
while(count < N):
els = driver.find_elements_by_css_selector("li.site-listing div.desc-container p.desc-paragraph a")
for el in els:
if(count < N):
t = el.get_attribute('innerHTML').lower()
fo = open(AD_FILE, "a")
fo.write(t + '\n')
fo.close()
count += 1
driver.find_element_by_css_selector("a.next").click()
def tearDown(self):
self.vdisplay.stop()
self.driver.quit()
def enable():
import time
import requests
import settings
from splinter import Browser
from xvfbwrapper import Xvfb
print "Trying to enable myself."
vdisplay = Xvfb()
vdisplay.start()
email = settings.getEmail()
password = settings.getPassword()
team_name = settings.getTeamName()
bot_user = settings.getBotUser()
browser = Browser('chrome')
url = 'https://{}.slack.com/services/{}'.format(team_name, bot_user)
browser.visit(url)
browser.fill('email', email)
browser.fill('password', password)
browser.find_by_id('signin_btn').first.click()
browser.find_link_by_text('Enable').first.click()
time.sleep(2) # Sometimes I saw a crash where there was no alert, so we'll wait a bit first.
alert = browser.get_alert()
alert.accept()
time.sleep(2) # If you close the display too quickly, the request doesn't get processed.
vdisplay.stop()
class PBSeleniumTest(unittest.TestCase):
def setUp(self):
env = os.environ
self.browser_bin = env.get("BROWSER_BIN", "") # o/w use WD's default
if "TRAVIS" in os.environ:
self.xvfb = 1
else:
# by default don't use XVFB if we are not running on CI
self.xvfb = int(env.get("ENABLE_XVFB", 0))
self.pb_ext_path = self.get_extension_path() # path to the extension
if self.xvfb:
self.vdisplay = Xvfb(width=1280, height=720)
self.vdisplay.start()
self.driver = self.get_chrome_driver()
self.js = self.driver.execute_script
def load_url(self, url, wait_on_site=0):
"""Load a URL and wait before returning."""
self.driver.get(url)
sleep(wait_on_site)
def get_extension_path(self):
"""Return the path to the extension to be tested."""
if "PB_EXT_PATH" in os.environ:
return os.environ["PB_EXT_PATH"]
else: # check the default path if PB_EXT_PATH env. variable is empty
print "Can't find the env. variable PB_EXT_PATH, will check ../.."
# if the PB_EXT_PATH environment variable is not set
# check the default location for the last modified crx file
exts = glob("../../*.crx") # get matching files
return max(exts, key=os.path.getctime) if exts else ""
def txt_by_css(self, css_selector, timeout=SEL_DEFAULT_WAIT_TIMEOUT):
"""Find an element by CSS selector and return it's text."""
return self.find_el_by_css(css_selector, timeout).text
def find_el_by_css(self, css_selector, timeout=SEL_DEFAULT_WAIT_TIMEOUT):
return WebDriverWait(self.driver, timeout).until(
EC.presence_of_element_located((By.CSS_SELECTOR, css_selector)))
def get_chrome_driver(self):
"""Setup and return a Chrom[e|ium] browser for Selenium."""
opts = Options()
absp = os.path.abspath
if "TRAVIS" in os.environ: # github.com/travis-ci/travis-ci/issues/938
opts.add_argument("--no-sandbox")
opts.add_extension(self.pb_ext_path) # will fail if ext can't be found
if self.browser_bin: # otherwise will use webdriver's default binary
print "Browser binary:", absp(self.browser_bin)
opts.binary_location = self.browser_bin # set binary location
# Fix for https://code.google.com/p/chromedriver/issues/detail?id=799
opts.add_experimental_option("excludeSwitches",
["ignore-certificate-errors"])
return webdriver.Chrome(chrome_options=opts)
def tearDown(self):
self.driver.quit()
if self.xvfb and self.vdisplay:
self.vdisplay.stop()
def test_stop(self):
orig_display = os.environ['DISPLAY']
xvfb = Xvfb()
xvfb.start()
self.assertNotEqual(orig_display, os.environ['DISPLAY'])
xvfb.stop()
self.assertIsNone(xvfb.proc)
self.assertEqual(orig_display, os.environ['DISPLAY'])
def xvfb(self, line, cell=None):
display = Xvfb(**self.xvfb_kwargs)
display.start()
if cell is None:
self.shell.ex(line)
else:
self.shell.ex(cell)
display.stop()
def test_with_xvfb():
if use_xvfb:
from xvfbwrapper import Xvfb
display = Xvfb(width=1920, height=1080)
display.start()
my_test()
if use_xvfb:
display.stop()
def set_up():
global browser
global xvfb
xvfb = Xvfb(width=1280, height=720, colordepth=24)
xvfb.start()
browser = webdriver.Chrome()
print browser.title
print ':%d' % xvfb.vdisplay_num
class Webdriver(unittest.TestCase):
def setUp(self):
self.vdisplay = Xvfb(width=1280, height=720)
self.vdisplay.start()
# if(not vdisplay.start()):
# fo = open(LOG_FILE, "a")
# fo.write("Xvfbfailure||"+str(TREATMENTID)+"||"+str(ID)+"\n")
# fo.close()
# sys.exit(0)
if(BROWSER=='firefox'):
if (platform.system()=='Darwin'):
self.driver = webdriver.Firefox()
elif (platform.system()=='Linux'):
self.driver = webdriver.Firefox(proxy=proxy)
else:
print "Unidentified Platform"
sys.exit(0)
elif(BROWSER=='chrome'):
print "WARNING: Expecting chromedriver at specified location !!"
if (platform.system()=='Darwin'):
chromedriver = "./experiment/chromedriver/chromedriver_mac"
os.environ["webdriver.chrome.driver"] = chromedriver
self.driver = webdriver.Chrome(executable_path=chromedriver)
elif (platform.system() == 'Linux'):
chromedriver = "./experiment/chromedriver/chromedriver_linux"
os.environ["webdriver.chrome.driver"] = chromedriver
chrome_option = webdriver.ChromeOptions()
chrome_option.add_argument("--proxy-server=yogi.pdl.cmu.edu:3128" )
self.driver = webdriver.Chrome(executable_path=chromedriver, chrome_options=chrome_option)
else:
print "Unidentified Platform"
sys.exit(0)
else:
print "Unsupported Browser"
sys.exit(0)
self.driver.implicitly_wait(10)
self.base_url = "https://www.google.com/"
self.verificationErrors = []
self.driver.set_page_load_timeout(40)
self.accept_next_alert = True
def test_webdriver(self):
driver = self.driver
cole.optIn(driver)
driver.get(SITE)
time.sleep(20)
#raw_input("wait")
pref = cole.get_ad_pref(driver)
print SITE
print 'pref=', pref
if pref != []:
fo = open(TARGET_FILE, "a")
fo.write(SITE+"||"+"@".join(pref)+'\n')
fo.close()
def tearDown(self):
self.vdisplay.stop()
self.driver.quit()
class T2w(unittest.TestCase):
def setUp(self):
p = helpers.findPort()
server = webctrl.myserver.myserver(host="localhost", port=p)
server.quiet = True
controllers = ctrl.setupControllers(False, True, True)
self.brewme = webctrl.runbrew(
controllers,
helpers.getTestRecipeList(),
server)
self.brewme.startNonBlocking()
print "up and running"
if VIRTUALDISPLAY:
self.vdisplay = Xvfb(width=1280, height=720)
self.vdisplay.start()
self.driver = webdriver.Firefox()
self.driver.implicitly_wait(30)
self.base_url = "http://*****:*****@name='recipe'])[12]").click()
driver.find_element_by_css_selector("input[type=\"submit\"]").click()
self.assertEqual("17 Falconers Flight IPA",
driver.find_element_by_css_selector("form > b").text)
print "===== SUCCESS test_recipeliststatus ====="
def is_element_present(self, how, what):
try:
self.driver.find_element(by=how, value=what)
except NoSuchElementException, e:
return False
return True
def test_start_without_existing_display(self):
del os.environ['DISPLAY']
xvfb = Xvfb()
self.addCleanup(xvfb.stop)
self.addCleanup(self.reset_display)
xvfb.start()
display_var = ':{}'.format(xvfb.new_display)
self.assertEqual(display_var, os.environ['DISPLAY'])
self.assertIsNotNone(xvfb.proc)
def xvfb(request):
from xvfbwrapper import Xvfb
vdisplay = Xvfb()
vdisplay.start()
yield
vdisplay.stop()
def test_renderer(self):
vdisplay = Xvfb()
vdisplay.start()
ren = window.Renderer()
window.record(ren, n_frames=1, out_path=self.out_file, size=(600, 600))
self.assertTrue(os.path.exists(self.out_file))
vdisplay.stop()
class OFMTransferSiteManager(OFMSiteManager):
def __init__(self, user=None): # pylint: disable=super-init-not-called
self.user = user
if self.user:
self._login_user = self.user.ofm_username
self._login_password = self.user.ofm_password
else:
self._login_user = os.environ('OFM_USERNAME')
self._login_password = os.environ('OFM_PASSWORD')
self.display = Xvfb()
self.display.start()
def download_transfer_excels(self, matchdays=None):
profile = webdriver.FirefoxProfile(os.path.join(BASE_DIR, 'ofm_transfer_data', 'firefox_profile'))
profile.set_preference("browser.download.dir", os.path.join(BASE_DIR, 'ofm_transfer_data'))
self.browser = webdriver.Firefox(firefox_profile=profile)
self.browser.set_page_load_timeout(10)
self.login()
if not matchdays:
matchdays = [Matchday.get_current()]
for matchday in matchdays:
if not self._is_transfer_file_present(matchday):
try:
self._jump_to_transfer_page(self, matchday=matchday) # pylint: disable=redundant-keyword-arg
except TimeoutError:
pass
@staticmethod
def _is_transfer_file_present(matchday=None):
if not matchday:
matchday = Matchday.get_current()
if os.path.isfile(os.path.join(BASE_DIR,
'ofm_transfer_data',
'ofm_spielerwechsel_{}_{}.csv'.format(
matchday.season.number,
matchday.number)
)):
return True
return False
@timeout(5, use_signals=False)
def _jump_to_transfer_page(self, matchday=None):
if not matchday:
self.jump_to_frame(Constants.Transfer.DOWNLOAD_TRANSFERS)
else:
self.jump_to_frame(Constants.Transfer.DOWNLOAD_TRANSFERS_FROM_MATCHDAY.format(matchday.number))
def kill_browser(self):
if self.browser:
self.browser.stop_client()
self.display.stop()
class Webdriver(unittest.TestCase):
def setUp(self):
self.vdisplay = Xvfb(width=1280, height=720)
self.vdisplay.start()
# if(not vdisplay.start()):
# fo = open(LOG_FILE, "a")
# fo.write("Xvfbfailure||"+str(TREATMENTID)+"||"+str(ID)+"\n")
# fo.close()
# sys.exit(0)
if(BROWSER=='firefox'):
if (platform.system()=='Darwin'):
self.driver = webdriver.Firefox()
elif (platform.system()=='Linux'):
self.driver = webdriver.Firefox(proxy=proxy)
else:
print "Unidentified Platform"
sys.exit(0)
elif(BROWSER=='chrome'):
print "WARNING: Expecting chromedriver at specified location !!"
if (platform.system()=='Darwin'):
chromedriver = "./experiment/chromedriver/chromedriver_mac"
os.environ["webdriver.chrome.driver"] = chromedriver
self.driver = webdriver.Chrome(executable_path=chromedriver)
elif (platform.system() == 'Linux'):
chromedriver = "./experiment/chromedriver/chromedriver_linux"
os.environ["webdriver.chrome.driver"] = chromedriver
chrome_option = webdriver.ChromeOptions()
chrome_option.add_argument("--proxy-server=yogi.pdl.cmu.edu:3128" )
self.driver = webdriver.Chrome(executable_path=chromedriver, chrome_options=chrome_option)
else:
print "Unidentified Platform"
sys.exit(0)
else:
print "Unsupported Browser"
sys.exit(0)
self.driver.implicitly_wait(10)
self.base_url = "https://www.google.com/"
self.verificationErrors = []
self.driver.set_page_load_timeout(40)
self.accept_next_alert = True
def test_webdriver(self):
driver = self.driver
helper.setLogFile(LOG_FILE)
helper.log("browserStarted||"+str(TREATMENTID), ID)
run = 0
while (run < RUNS):
helper.applyTreatment(driver, TREATMENTS[TREATMENTID], ID, TREATMENTID)
helper.wait_for_others(AGENTS, ID, ROUND)
time.sleep(20)
helper.collectMeasurement(driver, MEASUREMENT, ID, TREATMENTID)
run = run+1
def tearDown(self):
self.vdisplay.stop()
self.driver.quit()
def suro_ws_js_enabled_page():
vdisplay = Xvfb()
vdisplay.start()
browser = webdriver.Firefox()
browser.get('http://www.google.com')
print browser.title
browser.quit()
vdisplay.stop()
class T2w(unittest.TestCase):
def setUp(self):
p = helpers.findPort()
server = webctrl.myserver.myserver(host="localhost", port=p)
server.quiet = True
self.brewme = webctrl.runbrew(
helpers.timerCtrl(),
helpers.getSimpleBSMX(),
server)
self.brewme.startNonBlocking()
print "up and running"
# Comment out next two lines to see firefox on local display
self.vdisplay = Xvfb(width=1280, height=720)
self.vdisplay.start()
self.driver = webdriver.Firefox()
self.driver.implicitly_wait(30)
self.base_url = "http://localhost:%i" % p
self.verificationErrors = []
self.accept_next_alert = True
def url_base(self):
return(self.base_url)
def test_index(self):
driver = self.driver
self.driver.get(self.url_base())
self.assertTrue('Hopitty' in self.driver.title)
driver.find_element_by_css_selector("button").click()
self.assertEqual("Hopitty",
driver.find_element_by_css_selector("h1").text)
def test_RunControl(self):
driver = self.driver
self.driver.get(self.url_base())
# Make sure we start on home page
self.assertTrue('Hopitty' in self.driver.title)
driver.find_element_by_xpath("//a[2]/button").click()
self.assertEqual("Run Control",
driver.find_element_by_css_selector("h1").text)
def is_element_present(self, how, what):
try:
self.driver.find_element(by=how, value=what)
except NoSuchElementException, e:
return False
return True
def test_start_with_kwargs(self):
w = 800
h = 600
depth = 16
xvfb = Xvfb(width=w, height=h, colordepth=depth)
self.addCleanup(xvfb.stop)
xvfb.start()
self.assertEqual(w, xvfb.width)
self.assertEqual(h, xvfb.height)
self.assertEqual(depth, xvfb.colordepth)
self.assertEqual(os.environ['DISPLAY'], ':%d' % xvfb.vdisplay_num)
self.assertIsNotNone(xvfb.proc)
def getHtml2(url):
try:
vdisplay = Xvfb()
vdisplay.start()
browser = webdriver.Firefox()
browser.set_page_load_timeout(120)
browser.get(url)
return browser.page_source
except Exception as e:
print(e)
finally:
vdisplay.stop()
class web_gui:
"""Webgui lib."""
prefix = ""
def __init__(self,
output_dir=os.path.join(os.getcwd(), "results"),
**kwargs):
"""Instance initialisation.
To set the path for saving the gui page screenshots
To set the driver and display port as None for gui
initialisation
"""
self.output_dir = output_dir
self.default_delay = 20
self.driver = None
self.display = None
# this specified a prefix for the screenshots file names
# it cna be used to prepend the testcase name to the file name
def set_prefix(self, prefix=""):
"""Specify a prefix for the screenshots file name.
:param prefix - prefix string
:type prefix - string
"""
self.prefix = prefix
def _save_screenshot(self, name):
"""Save screenshot of the selenium web gui driver.
And save it in the current working directory of the os path.
:param name: name for saving scrrenshot
:type name: string
:return: full path of the file
:rtype: string
"""
full_path = os.path.join(self.output_dir, name)
self.driver.save_screenshot(full_path)
return full_path
def enter_input_value(self, key_id, input_value):
"""Enter_input_value method to Enter input value in a text box.
:param key_id: id of the gui element
:type key_id: string
:param input_value: input text
:type input_value: string
:raises assertion: Unable to enter value
"""
key_value = self.key[key_id]
select_value = resolv_dict(self.config, key_value)
self.scroll_view(select_value)
enter_value = enter_input(self.driver, select_value, input_value)
assert enter_value, "Unable to enter value %s" % input_value
def get_text(self, value):
"""To get the text from the text box in a gui page.
:param value: text value
:type value: string
:return: text to be captured
:rtype: string
"""
key_value = self.key[value]
key_value = eval("self.config" + key_value)
text = get_text_value(self.driver, key_value)
return text
def click_button(self, key_id):
"""Click_button method to click button in gui page.
:param key_id: id of the gui element
:type key_id: string
:raises assertion: Click button
"""
select_id = self.key[key_id]
select_key_value = resolv_dict(self.config, select_id)
self.scroll_view(select_key_value)
Click_button = click_button_id(self.driver, select_key_value)
assert Click_button is True, "Click button : %s " % Click_button
def selected_option_by_id(self, key_id, value):
"""Select option in the dropdown by id.
:param key_id: id of the gui element
:type key_id: string
:param value: element value
:type value: string
:raises assertion: Selecting dropdwon button
"""
select_id = self.key[key_id]
select_key_value = resolv_dict(self.config, select_id)
self.scroll_view(select_key_value)
select_button = select_option_by_id(self.driver, select_key_value,
value)
assert select_button, "Select button : %s " % select_button
def verify_radio(self, value):
"""Verify radio button in gui page.
:param value: radio id
:type value: string
:raises assertion: Changes are not applied properly
"""
key_value = self.key[value]
key_value = resolv_dict(self.config, key_value)
key_value = get_radio_button_value(self.driver, key_value)
assert key_value, "Changes are not applied properly"
def verify_drop_down(self, value, check_value):
"""Verify check value. Is present in the dropdown of the gui page.
:param value: dropdown id
:type value: string
:param check_value: check value from the dropdown
:type check_value: string
:return: True or False
:rtype: boolean
"""
key_value = self.key[value]
key_value = resolv_dict(self.config, key_value)
keyvalue = get_drop_down_value(self.driver, key_value)
if keyvalue == check_value:
return True
else:
return False
def verify_text(self, value, text):
"""Verify text value in gui page.
:param value: element id
:type value: string
:param text: text to be verified
:type text: string
"""
key_value = self.key[value]
key_value = resolv_dict(self.config, key_value)
key_value = get_text_value(self.driver, key_value)
if key_value == text:
return True
else:
return False
def scroll_view(self, scroll_value):
"""Scrolling into the particular option for better view.
:param scroll_value: gui element
:type scroll_value: string
"""
self.driver.execute_script(
"arguments[0].scrollIntoView();",
self.driver.find_element_by_id(scroll_value),
)
def _enter_text(self, txt_box, txt):
"""To enter the value in the text box.
:param txt_box: id of the text box
:type txt_box: string
:param txt: text to be entered
:type txt: string
"""
txt_list = list(txt)
for c in txt_list:
time.sleep(0.1)
txt_box.send_keys(c)
def check_element_visibility(self, *element, **kwargs):
"""To check the visibility of the element on the page.
:param element: id or name
:type element: string
:param kwargs: element
:type kwargs: string
:return: Web driver query output
"""
"""ex. *element=('id, 'element') or ('name, 'element')"""
timeout = kwargs.get("timeout", self.default_delay)
query = None
try:
query = WebDriverWait(self.driver, timeout).until(
EC.visibility_of_element_located(element))
except Exception:
print("check_element_visibility({}): timeout to find "
"element".format(element))
return query
def check_element_clickable(self, *element, **kwargs):
"""To check the element is clickable on the page.
:param element: id or name
:type element: string
:param kwargs: element
:type kwargs: string
:return: Web driver query output
"""
timeout = kwargs.get("timeout", self.default_delay)
query = None
try:
query = WebDriverWait(self.driver, timeout).until(
EC.element_to_be_clickable(element))
except Exception:
print("check_element_clickable({}): timeout to find "
"element".format(element))
return query
def check_element_selection_state_to_be(self, *element, **kwargs):
"""To check the element selection state on the page.
:param element: id or name
:type element: string
:param kwargs: element
:type kwargs: string
:return: Web driver query output
"""
timeout = kwargs.get("timeout", self.default_delay)
query = None
try:
query = WebDriverWait(self.driver, timeout).until(
EC.element_selection_state_to_be(element))
except Exception:
print("check_element_selection_state_to_be({}): timeout to find "
"element".format(element))
return query
def wait_for_element(self, index_by="id", ele_index=None):
"""Wait for element on the page.
:param index_id: 'id', defaults by id
:type index_id: string, optional
:param ele_index: gui element id, defaults by None
:type ele_index: string, optional
:raises assertion: check_element_visibility
"""
assert ele_index is not None, "ele_index=None"
if index_by == "name":
by = By.NAME
else:
by = By.ID
self.driver.implicitly_wait(self.default_delay)
print("wait_for_element(): check_element_visibility(%s, %s)" %
(str(by), ele_index))
ele = self.check_element_visibility(by, ele_index)
assert ele is not None, "check_element_visibility(%s, %s)=False" % (
str(by),
ele_index,
)
def check_element_exists(self, text):
"""To check if the elements exist in the gui page by xpath.
:param text: xpath of the element
:type text: string
:raises exception: If element not found
:return: Element or None
:rtype: string
"""
try:
element = self.driver.find_element_by_xpath(text)
print("Element '" + text + "' found")
return element
except NoSuchElementException:
print("check_element_exists No element '" + text + "' found")
return None
def check_element_exists_by_name(self, text):
"""To check if the elements exist in the gui page by name.
:param text: name of the element
:type text: string
:raises exception: If element not found
:return: Element or None
:rtype: string
"""
try:
element = self.driver.find_element_by_name(text)
print("Element '" + text + "' found")
return element
except NoSuchElementException:
print("No element '" + text + "' found")
return None
def wait_for_redirects(self):
"""Wait for possible redirects to settle down in gui page."""
# wait for possible redirects to settle down
url = self.driver.current_url
for _ in range(10):
time.sleep(5)
if url == self.driver.current_url:
break
url = self.driver.current_url
def gui_logout(self, id_value):
"""Logout of the gui page.
:param id_value: element id for logout
:type id_value: string
:raises exception: If element not found returns None
:return: True or False
:rtype: boolean
"""
try:
ele_botton = self.check_element_clickable(By.ID,
id_value,
timeout=3)
if ele_botton is not None:
ele_botton.click()
print("Logout clicked")
return True
except NoSuchElementException:
print("No logout botton element ('id', %s) found " % id_value)
return False
def driver_close(self, logout_id):
"""Logout of the gui page and close selenium web driver.
:param logout_id: element id for logout
:type logout_id: string
"""
self.gui_logout(logout_id)
self.driver.quit()
print("driver quit")
if self.display is not None:
self.display.stop()
print("display stop")
# Starts the python wrapper for Xvfb, Xephyr and Xvnc
# the backend can be set via BFT_OPTIONS
def open_display(self):
"""Display of the gui page after connecting through some random port."""
from pyvirtualdisplay.randomize import Randomizer
from boardfarm import config
if config.default_headless:
self.display = None
return
if config.default_display_backend == "xvnc":
xc, yc = config.default_display_backend_size.split("x")
x = int(xc)
y = int(yc)
r = Randomizer(
) if config.default_display_backend_port == 0 else None
self.display = Display(
backend=config.default_display_backend,
rfbport=config.default_display_backend_port,
rfbauth=os.environ["HOME"] + "/.vnc/passwd",
visible=0,
randomizer=r,
size=(x, y),
)
elif config.default_display_backend == "xvfb":
self.display = Xvfb()
else:
raise Exception("backend not yet tested!")
self.display.start()
def get_web_driver(self, proxy):
"""Get web driver using proxy.
:param proxy: proxy value of lan or wan, it can be obtained using
the method get_proxy(device)
:type proxy: web driver proxy
:raises Exception: Failed to get webproxy driver via proxy
:return: web driver
:rtype: web driver element
"""
from boardfarm import config
try:
self.driver = get_webproxy_driver(proxy, config)
except Exception:
traceback.print_exc()
raise Exception("Failed to get webproxy driver via proxy " + proxy)
return self.driver
def botton_click_to_next_page(self, index_by="id", ele_index=None):
"""Click button and verify.
:param index_by: 'id' or 'name'
:type index_by: string
:raises assertion: Assert if element index is None
:param ele_index: element index
:type ele_index: string
"""
assert ele_index is not None, "ele_index=None"
if index_by == "name":
by = By.NAME
else:
by = By.ID
# verify $botton is exist
botton = self.check_element_visibility(by, ele_index)
assert botton is not None, "timeout: not found %s in page" % ele_index
print("get botton value: %s" % botton.get_attribute("value"))
self._save_screenshot("%s_click.png" % ele_index)
self.check_element_clickable(by, ele_index).click()
def home_page(self, page_id):
"""Check the home page of the gui page.
:param page_id: id of the gui element
:type page_id: string
:raises assertion: timeout: not found home page
"""
home_page = self.check_element_visibility(By.ID, page_id)
# wait for possible redirects to settle down
self.wait_for_redirects()
time.sleep(10)
assert home_page is not None, "timeout: not found home page"
print(home_page.text)
self._save_screenshot(self.prefix + "home_page.png")
def navigate_to_target_page(self, navi_path):
"""Navigating to teh target page.
:param navi_path: navigation path of the page
:type navi_path: string
:raises assertion: Error in click
"""
for path in navi_path:
temp = self.key[path]
temp = resolv_dict(self.config, temp)
button = click_button_id(self.driver, temp)
assert button, "Error in click %s" % path
print("Click %s : PASS" % path)
time.sleep(2)
def __del__(self):
"""Destructor method.
Deletes the webgui object.
The function is called when the object references have gone.
"""
try:
self.display.stop()
except Exception:
pass
def run_inputs(instance_id, base_name, base_page):
print('Got bp: ' + str(base_page))
instance_status = {}
lp_status = 0
entered_chat = False
instance_status[str(base_name + str(instance_id))] = {}
instance_status[str(base_name + str(instance_id))]['timestamps'] = {}
instance_status[str(
base_name + str(instance_id))]['timestamps']['0_instance_born'] = str(
datetime.utcnow())
try:
if sys.platform != 'darwin' and use_xvfb:
print('Starting Xvfb')
vdisplay = Xvfb(width=1280, height=740)
vdisplay.start()
chrome_options = webdriver.ChromeOptions()
if sys.platform != 'darwin':
chrome_options.add_argument('headless')
chrome_options.add_argument('no-sandbox')
# chrome_options.add_argument('window-size=1200x700')
print('Will create driver')
driver = webdriver.Chrome(chrome_options=chrome_options)
print('driver:' + str(driver))
# os.environ['MOZ_HEADLESS'] = '1'
# driver = webdriver.Firefox()
except BaseException as e:
instance_status[str(base_name +
str(instance_id))]['status'] = 'could_not_open'
instance_status[str(base_name +
str(instance_id))]['status_message'] = str(e)
try:
if sys.platform != 'darwin' and use_xvfb:
print('Closing Xvfb')
vdisplay.stop()
except BaseException as e:
print(e)
return instance_status
try:
driver.get(base_page)
instance_status[
str(base_name +
str(instance_id))]['timestamps']['1_requested_website'] = str(
datetime.utcnow())
# Wait for chat launcher
WebDriverWait(driver, 45).until(
EC.visibility_of_element_located((By.ID, 'nds-chat-launcher')))
# Click on chat launcher
instance_status[str(
base_name +
str(instance_id))]['timestamps']['2_chat_became_available'] = str(
datetime.utcnow())
# print('will open chat')
sign_in_button = driver.find_element_by_id('nds-chat-launcher').click()
# print('flag 0')
# Move to iframe
WebDriverWait(driver, 30).until(
EC.visibility_of_element_located((By.ID, 'nds-chat-iframe')))
iframe = driver.find_element_by_id('nds-chat-iframe')
driver.switch_to_frame(iframe)
# Wait for segmento
# print('flag 1')
WebDriverWait(driver, 30, poll_frequency=0.1).until(
EC.visibility_of_element_located((By.ID, 'nds-chatbot-message-3')))
# print('flag 2')
WebDriverWait(driver, 15, poll_frequency=0.1).until(
EC.text_to_be_present_in_element(
(By.XPATH, '//*[@id="nds-chatbot-message-3"]/div[1]/div'),
'¿A qué segmento perteneces?'))
# print('flag 3')
instance_status[str(base_name + str(
instance_id))]['timestamps']['3_segmento_cliente_available'] = str(
datetime.utcnow())
# Click on segmento
driver.execute_script(
""" (function(e,s){e.src=s;e.onload=function(){jQuery.noConflict();console.log('jQuery 2.2.4 injected');jQuery('#nds-chatbot-message-3 > div.nds-chat-comment-option-wrap > div:nth-child(3)').click()};document.head.appendChild(e);})(document.createElement('script'),'//code.jquery.com/jquery-2.2.4.min.js') """
)
# print('will click segmento')
#clck_s = driver.find_element_by_xpath('//*[@id="nds-chatbot-message-3"]/div[2]/div[3]').click()
# print('Did click segmento')
instance_status[str(base_name + str(
instance_id))]['timestamps']['4_segmento_cliente_selected'] = str(
datetime.utcnow())
time.sleep(10)
# Wait for name question
WebDriverWait(driver, 15, poll_frequency=0.1).until(
EC.visibility_of_element_located((By.ID, 'nds-chatbot-message-4')))
WebDriverWait(driver, 15, poll_frequency=0.1).until(
EC.text_to_be_present_in_element(
(By.XPATH, '//*[@id="nds-chatbot-message-4"]/div/div'),
'¿Cómo te llamas?'))
instance_status[str(base_name + str(
instance_id))]['timestamps']['5_como_te_llamas_prompted'] = str(
datetime.utcnow())
# Get text input field and send name
input_field = driver.find_element_by_id('txMessage')
instance_name = base_name + str(instance_id)
input_field.send_keys(instance_name, Keys.ENTER)
instance_status[
str(base_name +
str(instance_id))]['timestamps']['6_instance_name_sent'] = str(
datetime.utcnow())
# Wait for "¿Qué puedo hacer por ti?" prompt
WebDriverWait(driver, 15, poll_frequency=0.1).until(
EC.visibility_of_element_located((By.ID, 'nds-chatbot-message-5')))
time.sleep(2)
# Send inputs and test responses:
inputs_to_send = {
'por que soy cliente advance':
"HSBC Advance es una propuesta para clientes distinguidos, enfocada a solucionar sus necesidades financieras, ofreciéndole productos y servicios con condiciones preferentes que te ayudarán a lograr tus metas de vida, así como la administración y construcción de tu patrimonio.",
'que significa hsbc':
"HSBC recibe su nombre de Hong Kong and Shanghai Banking Corporation Limited, compañía que se creó en 1865 para financiar el creciente comercio entre Europa, India y China.",
'cual es el telefono de empresarial':
'Para cualquier duda sobre productos o servicios empresariales por favor comunícate a nuestra Línea de Servicios Empresariales:'
} # Finish inputs_to_send
# 'tengo un afore':'HSBC y Principal Afore unen fuerzas para que al término de tu vida laboral tengas la mejor recompensa.',
entered_chat = True
for i_key in inputs_to_send:
input_field.send_keys(i_key, Keys.ENTER)
print('Sent ' + str(i_key))
input_time = datetime.utcnow()
instance_status[str(base_name + str(instance_id))]['timestamps'][
'sent_' + str(i_key.replace(' ', '_'))] = str(input_time)
waiting_for_res = True
time_out = 0
while waiting_for_res:
chat_nds_bubbles = driver.execute_script("""
var z = document.getElementsByClassName('nds-chat-comment-by-nds-chat');
var arr = Array.prototype.slice.call(z);
t = arr.map(function(e){return e.innerText});
return t
""")
for index, el in enumerate(chat_nds_bubbles):
if inputs_to_send[i_key] in el:
print('Got response for ' + str(i_key))
res_time = datetime.utcnow()
instance_status[str(
base_name + str(instance_id))]['timestamps'][
'got_response_for_' +
str(i_key.replace(' ', '_'))] = str(res_time)
waiting_for_res = False
break
time.sleep(1)
time_out += 1
if time_out > 60 * 3:
timed_out_timestamp = datetime.utcnow()
instance_status[str(
base_name + str(instance_id))]['status'] = 'timed_out'
instance_status[str(base_name + str(instance_id))][
'timestamps']['8_timed_out_timestamp'] = str(
timed_out_timestamp)
instance_status[str(
base_name + str(instance_id)
)]['timestamps']['chatbot_history'] = str(chat_nds_bubbles)
break
except BaseException as e:
print('Problems with instance ' + str(instance_id))
print(e)
if 'other_errors' in instance_status[str(base_name + str(
instance_id))]: # Concatenate previous errors if they exists
instance_status[str(base_name +
str(instance_id))]['other_errors'] += str(e)
else:
instance_status[str(base_name +
str(instance_id))]['other_errors'] = str(e)
if 'status' in instance_status[str(base_name + str(instance_id))]:
if instance_status[str(base_name +
str(instance_id))]['status'] != 'timed_out':
instance_status[str(base_name +
str(instance_id))]['status'] += '_correct*'
elif entered_chat:
instance_status[str(base_name +
str(instance_id))]['status'] = 'correct'
else:
instance_status[str(base_name +
str(instance_id))]['status'] = 'timed_out_2'
instance_status[
str(base_name +
str(instance_id))]['timestamps']['9_closing_instance'] = str(
datetime.utcnow())
print('Closing ' + base_name + str(instance_id))
try:
driver.close()
if sys.platform != 'darwin' and use_xvfb:
vdisplay.stop()
except BaseException as e:
instance_status[str(base_name +
str(instance_id))]['closing_error'] = str(e)
return instance_status
class WPTAgent(object):
"""Main agent workflow"""
def __init__(self, options, browsers):
from internal.browsers import Browsers
from internal.webpagetest import WebPageTest
from internal.traffic_shaping import TrafficShaper
from internal.adb import Adb
from internal.ios_device import iOSDevice
self.must_exit = False
self.options = options
self.capture_display = None
self.job = None
self.task = None
self.xvfb = None
self.root_path = os.path.abspath(os.path.dirname(__file__))
self.wpt = WebPageTest(options, os.path.join(self.root_path, "work"))
self.persistent_work_dir = self.wpt.get_persistent_dir()
self.adb = Adb(
self.options,
self.persistent_work_dir) if self.options.android else None
self.ios = iOSDevice(self.options.device) if self.options.iOS else None
self.browsers = Browsers(options, browsers, self.adb, self.ios)
self.shaper = TrafficShaper(options)
atexit.register(self.cleanup)
signal.signal(signal.SIGTERM, self.signal_handler)
signal.signal(signal.SIGINT, self.signal_handler)
def run_testing(self):
"""Main testing flow"""
import monotonic
start_time = monotonic.monotonic()
browser = None
exit_file = os.path.join(self.root_path, 'exit')
message_server = None
if not self.options.android and not self.options.iOS:
message_server = MessageServer()
message_server.start()
while not self.must_exit:
try:
if os.path.isfile(exit_file):
try:
os.remove(exit_file)
except Exception:
pass
self.must_exit = True
break
if message_server is not None and self.options.exit > 0 and \
not message_server.is_ok():
logging.error("Message server not responding, exiting")
break
if self.browsers.is_ready():
self.job = self.wpt.get_test()
if self.job is not None:
self.job['message_server'] = message_server
self.job['capture_display'] = self.capture_display
self.task = self.wpt.get_task(self.job)
while self.task is not None:
start = monotonic.monotonic()
try:
self.task['running_lighthouse'] = False
if self.job['type'] != 'lighthouse':
self.run_single_test()
if self.task['run'] == 1 and not self.task['cached'] and \
self.task['error'] is None and \
'lighthouse' in self.job and self.job['lighthouse']:
self.task['running_lighthouse'] = True
self.wpt.running_another_test(self.task)
self.run_single_test()
elapsed = monotonic.monotonic() - start
logging.debug('Test run time: %0.3f sec',
elapsed)
except Exception as err:
msg = ''
if err is not None and err.__str__(
) is not None:
msg = err.__str__()
self.task['error'] = 'Unhandled exception running test: '\
'{0}'.format(msg)
logging.exception(
"Unhandled exception running test: %s",
msg)
traceback.print_exc(file=sys.stdout)
self.wpt.upload_task_result(self.task)
# Set up for the next run
self.task = self.wpt.get_task(self.job)
if self.job is not None:
self.job = None
else:
self.sleep(self.options.polling)
except Exception as err:
msg = ''
if err is not None and err.__str__() is not None:
msg = err.__str__()
if self.task is not None:
self.task['error'] = 'Unhandled exception preparing test: '\
'{0}'.format(msg)
logging.exception("Unhandled exception: %s", msg)
traceback.print_exc(file=sys.stdout)
if browser is not None:
browser.on_stop_recording(None)
browser = None
if self.options.exit > 0:
run_time = (monotonic.monotonic() - start_time) / 60.0
if run_time > self.options.exit:
break
def run_single_test(self):
"""Run a single test run"""
self.alive()
browser = self.browsers.get_browser(self.job['browser'], self.job)
if browser is not None:
browser.prepare(self.job, self.task)
browser.launch(self.job, self.task)
if self.shaper.configure(self.job):
try:
if self.task['running_lighthouse']:
browser.run_lighthouse_test(self.task)
else:
browser.run_task(self.task)
except Exception as err:
msg = ''
if err is not None and err.__str__() is not None:
msg = err.__str__()
self.task['error'] = 'Unhandled exception in test run: '\
'{0}'.format(msg)
logging.exception("Unhandled exception in test run: %s",
msg)
traceback.print_exc(file=sys.stdout)
else:
self.task['error'] = "Error configuring traffic-shaping"
self.shaper.reset()
browser.stop(self.job, self.task)
# Delete the browser profile if needed
if self.task['cached'] or self.job['fvonly']:
browser.clear_profile(self.task)
else:
err = "Invalid browser - {0}".format(self.job['browser'])
logging.critical(err)
self.task['error'] = err
browser = None
def signal_handler(self, *_):
"""Ctrl+C handler"""
if self.must_exit:
exit(1)
if self.job is None:
print "Exiting..."
else:
print "Will exit after test completes. Hit Ctrl+C again to exit immediately"
self.must_exit = True
def cleanup(self):
"""Do any cleanup that needs to be run regardless of how we exit."""
logging.debug('Cleaning up')
self.shaper.remove()
if self.xvfb is not None:
self.xvfb.stop()
if self.adb is not None:
self.adb.stop()
if self.ios is not None:
self.ios.disconnect()
def sleep(self, seconds):
"""Sleep wrapped in an exception handler to properly deal with Ctrl+C"""
try:
time.sleep(seconds)
except IOError:
pass
def wait_for_idle(self, timeout=30):
"""Wait for the system to go idle for at least 2 seconds"""
import monotonic
import psutil
logging.debug("Waiting for Idle...")
cpu_count = psutil.cpu_count()
if cpu_count > 0:
target_pct = 50. / float(cpu_count)
idle_start = None
end_time = monotonic.monotonic() + timeout
idle = False
while not idle and monotonic.monotonic() < end_time:
self.alive()
check_start = monotonic.monotonic()
pct = psutil.cpu_percent(interval=0.5)
if pct <= target_pct:
if idle_start is None:
idle_start = check_start
if monotonic.monotonic() - idle_start > 2:
idle = True
else:
idle_start = None
def alive(self):
"""Touch a watchdog file indicating we are still alive"""
if self.options.alive:
with open(self.options.alive, 'a'):
os.utime(self.options.alive, None)
def startup(self):
"""Validate that all of the external dependencies are installed"""
ret = True
self.alive()
try:
import dns.resolver as _
except ImportError:
print "Missing dns module. Please run 'pip install dnspython'"
ret = False
try:
import monotonic as _
except ImportError:
print "Missing monotonic module. Please run 'pip install monotonic'"
ret = False
try:
from PIL import Image as _
except ImportError:
print "Missing PIL module. Please run 'pip install pillow'"
ret = False
try:
import psutil as _
except ImportError:
print "Missing psutil module. Please run 'pip install psutil'"
ret = False
try:
import requests as _
except ImportError:
print "Missing requests module. Please run 'pip install requests'"
ret = False
try:
subprocess.check_output(['python', '--version'])
except Exception:
print "Make sure python 2.7 is available in the path."
ret = False
try:
subprocess.check_output('convert -version', shell=True)
except Exception:
print "Missing convert utility. Please install ImageMagick " \
"and make sure it is in the path."
ret = False
try:
subprocess.check_output('mogrify -version', shell=True)
except Exception:
print "Missing mogrify utility. Please install ImageMagick " \
"and make sure it is in the path."
ret = False
# if we are on Linux and there is no display, enable xvfb by default
if platform.system() == "Linux" and not self.options.android and \
not self.options.iOS and 'DISPLAY' not in os.environ:
self.options.xvfb = True
if self.options.xvfb:
try:
from xvfbwrapper import Xvfb
self.xvfb = Xvfb(width=1920, height=1200, colordepth=24)
self.xvfb.start()
except ImportError:
print "Missing xvfbwrapper module. Please run 'pip install xvfbwrapper'"
ret = False
# Figure out which display to capture from
if platform.system() == "Linux" and 'DISPLAY' in os.environ:
logging.debug('Display: %s', os.environ['DISPLAY'])
self.capture_display = os.environ['DISPLAY']
elif platform.system() == "Darwin":
proc = subprocess.Popen(
'ffmpeg -f avfoundation -list_devices true -i ""',
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
_, err = proc.communicate()
for line in err.splitlines():
matches = re.search(r'\[(\d+)\] Capture screen', line)
if matches:
self.capture_display = matches.group(1)
break
elif platform.system() == "Windows":
self.capture_display = 'desktop'
if self.options.throttle:
try:
subprocess.check_output('sudo cgset -h', shell=True)
except Exception:
print "Missing cgroups, make sure cgroup-tools is installed."
ret = False
# Windows-specific imports
if platform.system() == "Windows":
try:
import win32api as _
import win32process as _
except ImportError:
print "Missing pywin32 module. Please run 'python -m pip install pypiwin32'"
ret = False
# Check the iOS install
if self.ios is not None:
ret = self.ios.check_install()
if not self.options.android and not self.options.iOS:
self.wait_for_idle(300)
self.shaper.remove()
if not self.shaper.install():
print "Error configuring traffic shaping, make sure it is installed."
ret = False
if self.adb is not None:
if not self.adb.start():
print "Error configuring adb. Make sure it is installed and in the path."
ret = False
return ret
class AMS(object):
def __init__(self, sleep=1.5):
""" sleep (float): add extra sleep if connection is slow
"""
self.download_dir = os.path.join(os.getcwd(), "tmp")
self.save_dir = os.path.join(os.getcwd(), "data")
self.sleep = sleep
def __enter__(self):
self.vdisplay = Xvfb()
self.vdisplay.start()
if not os.path.exists(self.download_dir):
os.makedirs(self.download_dir)
fp = webdriver.FirefoxProfile()
fp.set_preference("browser.download.folderList", 2)
fp.set_preference("browser.download.manager.showWhenStarting", False)
fp.set_preference("browser.download.dir", self.download_dir)
fp.set_preference("browser.helperApps.neverAsk.saveToDisk",
"application/octet-stream")
self.driver = webdriver.Firefox(firefox_profile=fp)
self.driver = make_patient(self.driver)
return self
def __exit__(self, *args):
self.driver.close()
self.vdisplay.stop()
def _open_page(self, url, attempt=0):
d = self.driver
def get_body():
try:
d.find_element_by_xpath("body")
except UnexpectedAlertPresentException:
try:
alert = d.switch_to.alert
alert.accept()
except NoAlertPresentException:
pass
except NoSuchElementException:
sleep(2)
pass
d.get(url)
attempt = 0
try:
print "trying to open page (%s)" % attempt
get_body()
except PageNotReady:
attempt = attempt + 1
if attempt < 10:
get_body()
def _open_start_page(self):
start_url = "http://mstatkommun.arbetsformedlingen.se/"
self._open_page(start_url)
def _select_municipality(self, municipality):
print "Select municipality (%s)" % municipality
def select_municpality():
muni_list.send_keys(municipality)
def click_municpality():
elem = self.driver.find_element_by_xpath_patiently(
"//div[@title='%s']/../.." % municipality)
elem.click()
muni_list = self.driver.find_element_by_xpath_patiently(
"//div[contains(@title,'Välj kommun')]")
muni_list.click()
patiently(select_municpality, StaleElementReferenceException)
patiently(click_municpality, NoSuchElementException)
def _select_year_month(self, year, month):
year_month = "%s-%02d" % (year, month)
print "Select year-month (%s)" % year_month
def click_month_list():
xpath = "//div[contains(@title,'Månad')]"
month_list = self.driver.find_elements_by_xpath_patiently(xpath)[1]
month_list.click()
sleep(self.sleep)
return month_list
#def select_year_month():
def click_year_month():
month_list.send_keys(year_month)
elem = self.driver.find_element_by_xpath_patiently(
"//div[@title='%s']" % year_month)
elem.click()
sleep(self.sleep)
month_list = patiently(click_month_list,
IndexError,
exception_to_raise=XPathError)
#patiently(select_year_month, (ElementNotVisibleException, StaleElementReferenceException))
patiently(click_year_month,
(ElementNotVisibleException, StaleElementReferenceException))
def _select_youth_only(self):
print "Select youth only"
def select_youth_only():
elem = self.driver.find_elements_by_xpath_patiently(
"//div[@title='18-24 år']")[-1]
elem.click()
sleep(self.sleep)
patiently(select_youth_only, NoSuchElementException)
def _select_foreign_only(self):
print "Select foreign only"
def select_foreign_only():
elem = self.driver.find_elements_by_xpath_patiently(
"//div[@title='Utrikesfödda']")[-1]
elem.click()
sleep(self.sleep)
patiently(select_foreign_only, NoSuchElementException)
def _download_file(self, xpath):
print "Download file"
d = self.driver
""" Downloads a file given an xpath to the download link
"""
file_list_before = os.listdir(self.download_dir)
# Download file
def download_file():
export_btn = d.find_element_by_xpath_patiently(xpath)
export_btn.click()
def get_filename():
file_list_after = os.listdir(self.download_dir)
diff = list(set(file_list_after) - set(file_list_before))
if len(diff) == 0:
raise DownloadError
else:
file_name = diff[0]
if ".part" in file_name:
raise DownloadError
else:
return file_name
patiently(download_file, NoSuchElementException, seconds=3)
file_name = patiently(get_filename, DownloadError)
return self.download_dir + "/" + file_name
def get_detailed(self,
municipality=None,
county=None,
year=None,
month=None,
youth_only=False,
foreign_only=False):
d = self.driver
self._open_start_page()
# Select municipality
if (municipality):
self._select_municipality(municipality)
# Select year-month
if year and month:
self._select_year_month(year, month)
if youth_only:
self._select_youth_only()
if foreign_only:
self._select_foreign_only()
downloaded_file = self._download_file(
"//td[contains(text(),'Exportera till Excel')]")
def parse_downloaded():
return DetailedTable().parse_downloaded_file(downloaded_file)
print downloaded_file
data = patiently(parse_downloaded, IndexError)
# Dismiss dialog
try:
close_btn = d.find_element_by_xpath_patiently(
"//button[text()='OK']")
close_btn.click()
except:
pass
return data
def _get_overview(self,
year=None,
month=None,
youth_only=False,
foreign_only=False):
data = OverviewTable()
d = self.driver
self._open_start_page()
sleep(self.sleep)
tab_elem = self.driver.find_element_by_xpath_patiently(
'//td[text()="Utskriftsrapporter"]')
tab_elem.click()
sleep(self.sleep)
# Select year-month
if year and month:
self._select_year_month(year, month)
if youth_only:
self._select_youth_only()
if foreign_only:
self._select_foreign_only()
def parse_downloaded():
return data.parse_downloaded_file(downloaded_file)
# [1] = "Arbetssökande antal/andel av befolkningen"
# [2] = "Arbetssökande antal/andel av den registerbaserade arbetskraften"
xpath = "(//td[contains(text(),'Exportera till Excel')])[2]"
downloaded_file = self._download_file(xpath)
data = patiently(parse_downloaded, IndexError)
data.verify(year=year,
month=month,
youth_only=youth_only,
foreign_only=foreign_only)
return data
def get_overview(self,
year=None,
month=None,
youth_only=False,
foreign_only=False):
data = []
while len(data) == 0:
try:
data = self._get_overview(year=year,
month=month,
youth_only=youth_only,
foreign_only=foreign_only)
except (StaleElementReferenceException, NoSuchElementException,
XPathError, DownloadError, DataValidationError,
EmptyFileError) as e:
""" A number of things could go wrong. If it does, just try again.
"""
print u"Caught an error (%s), but makes another attempt to get overview." % e.__class__.__name__
data = self._get_overview(year=year,
month=month,
youth_only=youth_only,
foreign_only=foreign_only)
return data
class SeleniumBot():
def __init__(self, username, password, num):
ensure_firefox_in_path()
self.num = num
self.xvfb = Xvfb(width=1280, height=720)
self.xvfb.start()
binary = FirefoxBinary(
'/u/rachanab/sum2017_research/firefox-bin/firefox')
self.browser = webdriver.Firefox(firefox_binary=binary)
# self.browser = webdriver.Firefox()
#log into certain bot
url = 'https://twitter.com/settings/account'
time.sleep(1)
self.browser.get(url)
time.sleep(4)
login1 = self.browser.find_element_by_class_name("js-username-field")
time.sleep(1)
login2 = self.browser.find_element_by_class_name("js-password-field")
time.sleep(1)
login1.send_keys(username)
time.sleep(1)
login2.send_keys(password)
time.sleep(1)
# submit button
self.browser.find_element_by_xpath(
"/html/body/div[1]/div[2]/div/div/div[1]/form/div[2]/button"
).click()
time.sleep(3)
def leave(self):
self.browser.quit()
self.xvfb.stop()
#https://stackoverflow.com/questions/20986631/how-can-i-scroll-a-web-page-using-selenium-webdriver-in-python/43299513
def _scroll_completely_down(self):
driver = self.browser
SCROLL_PAUSE_TIME = 0.5
# Get scroll height
last_height = driver.execute_script(
"return document.body.scrollHeight")
i = 0
while i < 2:
i = i + 1
# Scroll down to bottom
driver.execute_script(
"window.scrollTo(0, document.body.scrollHeight);")
# Wait to load page
time.sleep(SCROLL_PAUSE_TIME)
# Calculate new scroll height and compare with last scroll height
new_height = driver.execute_script(
"return document.body.scrollHeight")
if new_height == last_height:
break
last_height = new_height
def get_tweets(self):
self.browser.get('https://twitter.com')
#get all the tweets
tweets = self.browser.find_elements_by_class_name('tweet')
print(tweets)
print(len(tweets))
text = []
for i in range(min(len(tweets), 15)):
text.append(tweets[i].text)
print(text)
def retweet(self):
self.browser.get('https://twitter.com')
#get all the tweets
# tweets = self.browser.find_element_by_css_selector('#stream-item-tweet-935947628092841984 > div:nth-child(1) > div:nth-child(2) > div:nth-child(5) > div:nth-child(2) > div:nth-child(2) > button:nth-child(1)')
tweet = self.browser.find_element_by_css_selector(
".js-stream-item .ProfileTweet-action--retweet div")
time.sleep(3)
tweet.click()
time.sleep(3)
self.browser.save_screenshot("screenshotretweet_3.png")
confirm_button = self.browser.find_element_by_class_name(
'EdgeButton.EdgeButton--primary.retweet-action')
time.sleep(3)
confirm_button.click()
self.browser.save_screenshot("screenshot2retweet_3.png")
def tag_retweet(self, flag, sample_num):
if flag == 'c':
keywords = credentials.conservative
elif flag == 'l':
keywords = credentials.liberal
else:
print("invalid flag")
exit()
container = self.browser.find_element_by_id("stream-items-id")
tweets = container.find_elements_by_css_selector(
".tweet.js-stream-tweet.js-actionable-tweet.js-profile-popup-actionable"
)
retweets = container.find_elements_by_css_selector(
'.ProfileTweet-actionButton.js-actionButton.js-actionRetweet')
# get the retweet buttons and click a random sample of them
i = i - sample_num / 2
cnt = 0
while cnt < min(len(retweets), sample_num):
try:
if any(word in tweets[i].text for word in
keywords) and "Promoted" not in tweets[i].text:
time.sleep(1)
retweets[i].click()
print(tweets[i].text)
time.sleep(5.0)
button = retweets[i].find_element_by_xpath(
'//*[@id="retweet-tweet-dialog-dialog"]/div[2]/form/div[2]/div[3]/button'
)
button.click()
cnt += 1
print("clicked retweet dialogue")
except Exception as e:
print(Exception)
# print(str(i) + " button was " + retweets[i].text)
i += 1
#interior part of button
def follow_user(self, username):
self.browser.get('https://twitter.com/' + username)
time.sleep(4)
#a = self.browser.find_element_by_xpath("/html/body/div[2]/div[2]/div/div[1]/div/div[2]/div/div/div[2]/div/div/ul/li[7]/div/div/span[2]/button[1]")
a = self.browser.find_element_by_class_name(
"EdgeButton.EdgeButton--secondary.EdgeButton--medium.button-text.follow-text"
)
time.sleep(3)
while True:
print(a.text)
a.click()
time.sleep(3)
return
def click_on_link(self):
'''
self.browser.get('https://twitter.com/')
time.sleep(3)
self.browser.save_screenshot("screenshotscrolldown.png")
a = self.browser.find_elements_by_class_name("js-stream-item.stream-item.stream-item")
print(len(a))
a[0].click()
time.sleep(3)
'''
self.browser.get(
"https://twitter.com/senorrinhatch/status/939672194271072256")
time.sleep(20)
self._scroll_completely_down()
time.sleep(20)
self.browser.save_screenshot("screenshotlink.png")
#a = self.browser.find_element_by_class_name("TwitterCardsGrid.TwitterCard.TwitterCard--animation")
#c = self.browser.find_element_by_class_name("TwitterCardsGrid-col--12.TwitterCardsGrid-col--spacerBottom.CardContent")
#d = c.find_element_by_css_selector("js-openLink.u-block.TwitterCardsGrid-col--12.TwitterCard-container.TwitterCard-container--clickable.SummaryCard--large")
#d = self.browser.find_element_by_xpath("/html/body/div/div/a")
# d = self.browser.find_element_by_class_name("u-block")
d = self.browser.find_elements_by_tag_name("iframe")
print(len(d))
time.sleep(3)
for iframe in d:
print(iframe.get_attribute("id"))
print(iframe.get_attribute("src"))
time.sleep(3)
self.browser.get(
"https://twitter.com/i/cards/tfw/v1/939672194271072256?cardname=summary_large_image&autoplay_disabled=true&earned=true&edge=true&lang=en&card_height=344&scribe_context=%7B%22client%22%3A%22web%22%2C%22page%22%3A%22permalink%22%2C%22section%22%3A%22permalink%22%2C%22component%22%3A%22tweet%22%7D&bearer_token=AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%252F40K4moUkGsoc%253DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw#xdm_e=https%3A%2F%2Ftwitter.com&xdm_c=default910&xdm_p=1"
)
time.sleep(3)
self.browser.save_screenshot("clicked_link.png")
#this works but its every other; also cant trust last one
def find_name(self):
self.browser.get('https://twitter.com')
#get all the tweets
tweets = self.browser.find_elements_by_class_name('tweet')
for i in range(min(len(tweets), 15)):
print(tweets[i].text)
username = tweets[i].get_attribute('data-retweeter')
if not username:
username = tweets[i].get_attribute('data-screen-name')
print("####")
print(username)
print("####")
#this works for retweets only
def find_name_2(self):
self.browser.get('https://twitter.com/')
a = self.browser.find_elements_by_class_name(
"pretty-link.js-user-profile-link")
for b in a:
print(b.get_attribute("href"))
#TwitterCard-title.js-cardClick.tcu-textEllipse--multiline
# js-openLink u-block TwitterCardsGrid-col--12 TwitterCard-container TwitterCard-container--clickable SummaryCard--large
def follow_users(self):
print("Follow Users")
client = MongoClient()
db = client.poldatabase
collection = db.followees
a = collection.find()
i = 0
for b in a:
i = i + 1
if self.num not in b['bots']:
print(i)
#print(b['name'])
self.follow_user(b['name'])
b['bots'] = b['bots'] + [self.num]
collection.update({'_id': b['_id']}, b)
time.sleep(random.randrange(5, 10))
class Site:
def __init__(self, args):
self.args = args
self.site_name = type(self).__name__
self.site_displayname = BashColor.HEADER + BashColor.BOLD + self.site_name + BashColor.END \
if sys.stdout.isatty() else self.site_name
self.config = ConfigParser()
self.__read_config_file('credentials.cfg.orig')
self.__read_config_file('credentials.cfg')
self._parse_credentials()
self.LOGIN_PAGE = self._get_login_page_url()
self._init_browser()
self._parse_configuration()
def __read_config_file(self, filename):
self.config.read(os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, filename)))
def _parse_credentials(self):
if os.environ.get(self.site_name.upper() + '_USERNAME'):
self.USERNAME = os.environ.get(self.site_name.upper() + '_USERNAME')
else:
self.USERNAME = self.config[self.site_name]['USERNAME']
if os.environ.get(self.site_name.upper() + '_PASSWORD'):
self.PASSWORD = os.environ.get(self.site_name.upper() + '_PASSWORD')
else:
self.PASSWORD = self.config[self.site_name]['PASSWORD']
def _get_login_page_url(self):
raise NotImplementedError("This is not the implementation you are looking for.")
def _parse_configuration(self):
# this method should be overwritten by a site, if there are more configs to parse than just the credentials
# and for things which need a running browser
pass
def _init_browser(self):
if self.args and not self.args.show_browser:
self.display = Xvfb()
self.display.start()
if self.args and self.args.verbose and self.args.verbose >= 3:
log_level = 'trace'
elif self.args and self.args.verbose and self.args.verbose == 2:
log_level = 'debug'
elif self.args and self.args.verbose and self.args.verbose == 1:
log_level = 'info'
else:
log_level = 'warn'
capabilities = DesiredCapabilities.FIREFOX.copy()
capabilities["moz:firefoxOptions"] = {
"log": {
"level": log_level,
},
}
options = Options()
options.log.level = log_level
profile = FirefoxProfile()
profile.set_preference("browser.download.folderList", 2)
profile.set_preference("browser.download.manager.showWhenStarting", False)
profile.set_preference("browser.download.dir", EXPORTS_FOLDER)
profile.set_preference("browser.helperApps.neverAsk.saveToDisk", "text/csv, application/zip")
profile.set_preference("browser.helperApps.alwaysAsk.force", False)
profile.set_preference("devtools.jsonview.enabled", False)
profile.set_preference("media.volume_scale", "0.0")
# https://github.com/mozilla/geckodriver/issues/858#issuecomment-322512336
profile.set_preference("dom.file.createInChild", True)
self.browser = Firefox(
firefox_profile=profile,
capabilities=capabilities,
firefox_options=options,
log_path="{timestamp}_geckodriver.log".format(timestamp=TIMESTAMP)
)
# http://stackoverflow.com/questions/42754877/cant-upload-file-using-selenium-with-python-post-post-session-b90ee4c1-ef51-4 # pylint: disable=line-too-long
self.browser._is_remote = False # pylint: disable=protected-access
self.login()
def login(self):
sys.stdout.write('===== {site_displayname}: performing login'.format(site_displayname=self.site_displayname))
sys.stdout.flush()
self.browser.get(self.LOGIN_PAGE)
time.sleep(1)
iteration = 0
while self._user_is_not_logged_in():
iteration += 1
try:
self._insert_login_credentials()
self._click_login_button()
except NoSuchElementException as e:
if iteration > 10:
raise e
time.sleep(iteration * 1)
continue
self._handle_captcha_challenge_if_present()
if iteration > 2:
self._handle_login_unsuccessful()
def _handle_captcha_challenge_if_present(self):
# to be implemented for sites with captchas individually
pass
def _handle_login_unsuccessful(self):
time.sleep(1)
if self._user_is_not_logged_in():
command_line.error("Login to {site_name} failed.".format(site_name=self.site_name))
sys.stdout.write("Please check if the credentials are correctly set in your credentials.cfg\r\n")
sys.stdout.flush()
self.kill_browser()
sys.exit(1)
def _user_is_not_logged_in(self):
return len(self.browser.find_elements_by_xpath(self.LOGIN_BUTTON_SELECTOR)) > 0 \
and len(self.browser.find_elements_by_xpath(self.LOGIN_USERNAME_SELECTOR)) > 0 \
and len(self.browser.find_elements_by_xpath(self.LOGIN_PASSWORD_SELECTOR)) > 0
def _insert_login_credentials(self):
login_field_user = self.browser.find_element_by_xpath(self.LOGIN_USERNAME_SELECTOR)
login_field_user.clear()
login_field_user.send_keys(self.USERNAME)
login_field_password = self.browser.find_element_by_xpath(self.LOGIN_PASSWORD_SELECTOR)
login_field_password.clear()
login_field_password.send_keys(self.PASSWORD)
def _click_login_button(self):
login_button = self.browser.find_element_by_xpath(self.LOGIN_BUTTON_SELECTOR)
login_button.click()
time.sleep(2) # wait for page to load
def kill_browser(self):
self.browser.stop_client()
self.browser.close()
try:
self.browser.quit()
except WebDriverException:
pass
if self.args and not self.args.show_browser:
self.display.stop()
def get_json_from_html(self):
response = self.browser.find_element_by_tag_name("pre").text.strip()
return json.loads(response)
def evaluate_nodes(self, nodes, uncertain_parameters):
"""
Evaluate the the model and calculate the features
for the nodes (values) for the uncertain parameters.
Parameters
----------
nodes : array
The values for the uncertain parameters
to evaluate the model and features for.
uncertain_parameters : list
A list of the names of all uncertain parameters.
Returns
-------
results : list
A list where each element is a result dictionary for each set
of model evaluations.
An example:
.. code-block:: Python
result = {self.model.name: {"values": array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]),
"time": array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])},
"feature1d": {"values": array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]),
"time": array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])},
"feature0d": {"values": 1,
"time": np.nan},
"feature2d": {"values": array([[0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]]),
"time": array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])},
"feature_adaptive": {"values": array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]),
"time": array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]),
"interpolation": scipy interpolation object},
"feature_invalid": {"values": np.nan,
"time": np.nan}}
results = [result 1, result 2, ..., result N]
Raises
------
ImportError
If xvfbwrapper is not installed.
"""
if self.model.suppress_graphics:
if not prerequisites:
raise ImportError(
"Running with suppress_graphics require: xvfbwrapper")
vdisplay = Xvfb()
vdisplay.start()
results = []
model_parameters = self.create_model_parameters(
nodes, uncertain_parameters)
if self.CPUs:
import multiprocess as mp
pool = mp.Pool(processes=self.CPUs)
# pool.map(self._parallel.run, model_parameters)
# chunksize = int(np.ceil(len(model_parameters)/self.CPUs))
chunksize = 1
for result in tqdm(pool.imap(self._parallel.run, model_parameters,
chunksize),
desc="Running model",
total=len(nodes.T)):
results.append(result)
pool.close()
else:
for result in tqdm(imap(self._parallel.run, model_parameters),
desc="Running model",
total=len(nodes.T)):
results.append(result)
if self.model.suppress_graphics:
vdisplay.stop()
return results
def run():
print('Sreencast website animation')
xvfb = Xvfb(width=1280, height=720, colordepth=24)
xvfb.start()
url = "https://test2.airmeet.com/event/session?t=b8f27257-8082-4a16-b1f9-851ba1e982e8"
chromeOptions = webdriver.ChromeOptions()
chromeOptions.add_argument("--kiosk")
chromeOptions.add_argument("--window-size=1280x720")
chromeOptions.add_argument("disable-infobars")
chromeOptions.add_argument("--no-sandbox")
chromeOptions.add_argument("--use-fake-ui-for-media-stream")
# chromeOptions.add_argument('--headless')
chromeOptions.add_argument("--autoplay-policy=no-user-gesture-required")
chromeOptions.add_experimental_option(
'prefs', {
'credentials_enable_service': False,
'profile': {
'password_manager_enabled': False
}
})
print('Starting browser')
browser = webdriver.Chrome(chrome_driver_path,
chrome_options=chromeOptions)
print('Getting url')
browser.get(url)
print('got url')
time.sleep(5)
print('Slept 5 secs')
# ffmpeg_stream = 'ffmpeg -y -r 30 -f x11grab -s 1280x720 -i :%d+nomouse -f pulse -ac 2 -i default -c:v libx264rgb -crf 15 -preset:v ultrafast -c:a pcm_s16le -af aresample=async=1:first_pts=0 /tmp/out.mkv' % xvfb.new_display
ffmpeg_stream = 'ffmpeg -y -r 24 -f x11grab -s 1280x720 -i :%d+nomouse -f pulse -ac 2 -i default -c:v libx264rgb -crf 28 -preset:v ultrafast -c:a pcm_s16le -af aresample=async=1:first_pts=0 /tmp/out.mkv' % xvfb.new_display
# ffmpeg_stream = 'ffmpeg -y -r 24 -f x11grab -s 640x360 -i :%d+nomouse -f pulse -ac 2 -i default -c:v libx264 -preset superfast -acodec libmp3lame -ar 48000 -pix_fmt yuv420p -threads 0 -f matroska /tmp/out.mkv' % xvfb.new_display
# os.system(ffmpeg_stream)
print("Execute this command")
print(ffmpeg_stream)
args = shlex.split(ffmpeg_stream)
p = subprocess.Popen(args)
print(p)
time.sleep(2)
try:
time.sleep(1)
browser.find_elements_by_class_name('broadcast-streams')[0].click()
# time.sleep(0.5)
# browser.find_elements_by_class_name('ml-1')[0].click()
# time.sleep(1)
# name = browser.find_element_by_name('name')
# print(name)
# name.send_keys("Selen")
# time.sleep(1)
# designation = browser.find_element_by_name('designation')
# print(designation)
# designation.send_keys("Selen")
# time.sleep(1)
# company = browser.find_element_by_name('company')
# print(company)
# company.send_keys("Selen")
# time.sleep(1)
# browser.find_element_by_id('city').send_keys("Selen")
# time.sleep(0.2)
# browser.find_element_by_id('country').send_keys("Selen")
# time.sleep(0.2)
# browser.find_elements_by_class_name('w-100')[1].click()
# time.sleep(0.2)
# browser.find_elements_by_class_name('w-100')[1].click()
# time.sleep(30)
# browser.find_elements_by_class_name('inverted')[0].click()
except Exception as e:
print("Some error occurred")
traceback.print_exc()
# ffmpeg_stream = 'ffmpeg -y -f x11grab -s 1344x756 -r 30 -i :%d+nomouse -f pulse -ac 2 -i default -c:v libx264 -preset superfast -acodec libmp3lame -ar 48000 -pix_fmt yuv420p -s 1344x756 -threads 0 -f matroska "%s.mkv"' % (xvfb.new_display, dir_path + "/" + recording_title)
# args = shlex.split(ffmpeg_stream)
# print(ffmpeg_stream)
time.sleep(25) # record for 30 secs
p.terminate()
browser.quit()
xvfb.stop()
class TestLifecycle(unittest.TestCase):
def test_create_destroy(self):
thread_names = lambda: [ t.name for t in threading.enumerate() ]
self.assertNotIn('MPVEventHandlerThread', thread_names())
m = mpv.MPV()
self.assertIn('MPVEventHandlerThread', thread_names())
m.terminate()
self.assertNotIn('MPVEventHandlerThread', thread_names())
def test_flags(self):
with self.assertRaises(AttributeError):
mpv.MPV('this-option-does-not-exist')
m = mpv.MPV('cursor-autohide-fs-only', 'fs', video=False)
self.assertTrue(m.fullscreen)
self.assertEqual(m.cursor_autohide, 1000)
m.terminate()
def test_options(self):
with self.assertRaises(AttributeError):
mpv.MPV(this_option_does_not_exists=23)
m = mpv.MPV(osd_level=0, loop='inf', deinterlace=False)
self.assertEqual(m.osd_level, 0)
# For compatibility with mpv master (v0.32.0-585-gfba1c681b8) accept both
self.assertIn(m.loop, ['inf', True])
self.assertEqual(m.deinterlace, False)
m.terminate()
def test_event_callback(self):
handler = mock.Mock()
m = mpv.MPV(video=False)
def cb(evt):
handler(evt.as_dict(decoder=mpv.lazy_decoder))
m.register_event_callback(cb)
m.play(TESTVID)
m.wait_for_playback()
m.unregister_event_callback(cb)
handler.assert_has_calls([
mock.call({'event': 'start-file', 'playlist_entry_id': 1}),
mock.call({'event': 'end-file', 'reason': 'error', 'playlist_entry_id': 1, 'file_error': 'no audio or video data played'})
], any_order=True)
handler.reset_mock()
m.terminate()
handler.assert_not_called()
def test_wait_for_property_negative(self):
self.disp = Xvfb()
self.disp.start()
m = mpv.MPV(vo=testvo)
m.play(TESTVID)
result = Future()
def run():
nonlocal self
result.set_running_or_notify_cancel()
try:
m.wait_for_property('mute')
result.set_result(False)
except mpv.ShutdownError:
result.set_result(True)
t = threading.Thread(target=run, daemon=True)
t.start()
time.sleep(1)
m.terminate()
t.join()
self.disp.stop()
assert result.result()
def test_wait_for_property_positive(self):
self.disp = Xvfb()
self.disp.start()
handler = mock.Mock()
m = mpv.MPV(vo=testvo)
m.play(TESTVID)
def run():
nonlocal self
m.wait_for_property('mute')
handler()
t = threading.Thread(target=run, daemon=True)
t.start()
m.wait_until_playing(timeout=2)
m.mute = True
t.join()
m.terminate()
handler.assert_called()
self.disp.stop()
def test_wait_for_event(self):
self.disp = Xvfb()
self.disp.start()
handler = mock.Mock()
m = mpv.MPV(vo=testvo)
m.play(TESTVID)
result = Future()
def run():
nonlocal self
result.set_running_or_notify_cancel()
try:
m.wait_for_event('seek')
result.set_result(False)
except mpv.ShutdownError:
result.set_result(True)
t = threading.Thread(target=run, daemon=True)
t.start()
time.sleep(1)
m.terminate()
t.join()
self.disp.stop()
assert result.result()
def test_wait_for_property_shutdown(self):
self.disp = Xvfb()
self.disp.start()
handler = mock.Mock()
m = mpv.MPV(vo=testvo)
m.play(TESTVID)
with self.assertRaises(mpv.ShutdownError):
# level_sensitive=false needed to prevent get_property on dead
# handle
with m.prepare_and_wait_for_property('mute', level_sensitive=False):
m.terminate()
self.disp.stop()
def test_wait_for_property_event_overflow(self):
self.disp = Xvfb()
self.disp.start()
handler = mock.Mock()
m = mpv.MPV(vo=testvo)
m.play(TESTVID)
with self.assertRaises(mpv.EventOverflowError):
# level_sensitive=false needed to prevent get_property on dead
# handle
with m.prepare_and_wait_for_property('mute', cond=lambda val: time.sleep(0.001)):
for i in range(10000):
try:
# We really have to try hard to fill up the queue here. Simple async commands will not work,
# since then command_async will throw a memory error first. Property changes also do not work,
# since they are only processsed when the event loop is idle. This here works reliably.
m.command_async('script-message', 'foo', 'bar')
except:
pass
self.disp.stop()
def test_wait_for_event_shutdown(self):
self.disp = Xvfb()
self.disp.start()
m = mpv.MPV(vo=testvo)
m.play(TESTVID)
with self.assertRaises(mpv.ShutdownError):
with m.prepare_and_wait_for_event('seek'):
m.terminate()
self.disp.stop()
def test_wait_for_shutdown(self):
self.disp = Xvfb()
self.disp.start()
handler = mock.Mock()
m = mpv.MPV(vo=testvo)
m.play(TESTVID)
with self.assertRaises(mpv.ShutdownError):
with m.prepare_and_wait_for_event(None) as result:
m.terminate()
result.result()
self.disp.stop()
def test_log_handler(self):
handler = mock.Mock()
self.disp = Xvfb()
self.disp.start()
m = mpv.MPV(vo=testvo, log_handler=handler)
m.play(TESTVID)
# Wait for playback to start
m.wait_until_playing(timeout=2)
m.command("print-text", 'This is a python-mpv test')
m.wait_for_playback()
m.terminate()
for call in handler.mock_calls:
_1, (a, b, c), _2 = call
if a == 'info' and b == 'cplayer' and 'This is a python-mpv test' in c:
break
else:
self.fail('"Test log entry not found in log handler calls: '+','.join(repr(call) for call in handler.mock_calls))
self.disp.stop()
class DownloadExpirationsList:
def __init__(self):
self.driver = None
self.vdisplay = Xvfb()
def _display_start(self):
self._display_stop()
try:
self.vdisplay.start()
except:
print('Start without virtual display')
def _display_stop(self):
try:
self.vdisplay.stop()
except:
print('Virtual display can not stoped')
def _webdriver_start(self):
fp = webdriver.FirefoxProfile()
fp.set_preference("browser.download.folderList", 2)
fp.set_preference("browser.download.manager.showWhenStarting", False)
fp.set_preference("browser.download.dir", EXP_LIST_DIR_PATH)
fp.set_preference("browser.helperApps.neverAsk.saveToDisk",
"application/csv")
fp.set_preference("plugin.disable_full_page_plugin_for_types",
"application/csv")
driver = webdriver.Firefox(fp)
driver.maximize_window()
driver.implicitly_wait(15)
self.driver = driver
def _webdriver_stop(self):
try:
self.driver.quit()
except WebDriverException:
print('Ceckodriver can not stoped')
def open(self):
self._display_start()
self._webdriver_start()
def close(self):
self._webdriver_stop()
self._display_stop()
def download(self):
bar = IncrementalBar('Downloading ', max=10)
self.driver.get(CME_LINK +
'/tools-information/quikstrike/options-calendar.html')
first_window = self.driver.window_handles[0]
bar.next()
sleep(5)
self.driver.get(
CME_TOOLS_LINK +
'/User/QuikStrikeView.aspx?viewitemid=IntegratedCMEOptionExpirationCalendar'
)
bar.next()
sleep(5)
self.driver.find_element_by_xpath(
'//a[@id="MainContent_ucViewControl_IntegratedCMEOptionExpirationCalendar_ucViewControl_hlCMEProducts"]'
).click()
bar.next()
sleep(5)
for handle in self.driver.window_handles:
if handle != first_window:
self.driver.switch_to_window(handle)
self.driver.find_element_by_xpath(
'//a[@id="ctl00_cphMain_lvTabs_ctrl3_lbTab"]').click()
bar.next()
sleep(3)
self.driver.find_element_by_xpath(
'//a[@id="cphMain_ucProductBrowser_ucProductFilter_ucTrigger_lnkTrigger"]'
).click()
bar.next()
sleep(3)
self.driver.find_element_by_xpath(
'//input[@id="cphMain_ucProductBrowser_ucProductFilter_ucGroupList_rblGroups_4"]'
).click()
bar.next()
sleep(3)
self.driver.find_element_by_xpath(
'//input[@id="cphMain_ucProductBrowser_ucProductFilter_ucContractTypeList_rblContractType_1"]'
).click()
bar.next()
sleep(3)
self.driver.find_element_by_xpath(
'//input[@id="cphMain_ucProductBrowser_ucProductFilter_btnApply"]'
).click()
bar.next()
sleep(3)
self.driver.find_element_by_xpath(
'//a[@id="cphMain_ucProductBrowser_ucProductActions_ucTrigger_lnkTrigger"]'
).click()
bar.next()
sleep(3)
self.driver.find_element_by_xpath(
'//a[@id="cphMain_ucProductBrowser_ucProductActions_lnkExport"]'
).click()
bar.next()
# self.driver.find_element_by_xpath(
# '//a[@id="cphMain_ucProductBrowser_ucProductActions_lnkShowExpirations"]').click()
# bar.next()
# sleep(4)
# iframe = self.driver.find_element_by_xpath('//iframe[@id="mainFrame"]')
# self.driver.switch_to_frame(iframe)
# bar.next()
# sleep(4)
# self.driver.find_element_by_xpath('//a[@id="ctl03_ucExport_lnkTrigger"]').click()
# bar.next()
sleep(5)
bar.finish()
class NipypeConfig(object):
"""Base nipype config class"""
def __init__(self, *args, **kwargs):
self._config = configparser.ConfigParser()
self._cwd = None
config_dir = os.path.expanduser(
os.getenv("NIPYPE_CONFIG_DIR", default="~/.nipype"))
self.data_file = os.path.join(config_dir, "nipype.json")
self.set_default_config()
self._display = None
self._resource_monitor = None
self._config.read(
[os.path.join(config_dir, "nipype.cfg"), "nipype.cfg"])
for option in CONFIG_DEPRECATIONS:
for section in ["execution", "logging", "monitoring"]:
if self.has_option(section, option):
new_section, new_option = CONFIG_DEPRECATIONS[option][
0].split(".")
if not self.has_option(new_section, new_option):
# Warn implicit in get
self.set(new_section, new_option,
self.get(section, option))
@property
def cwd(self):
"""Cache current working directory ASAP"""
# Run getcwd only once, preventing multiproc to finish
# with error having changed to the wrong path
if self._cwd is None:
try:
self._cwd = os.getcwd()
except OSError:
warn(
'Trying to run Nipype from a nonexistent directory "{}".'.
format(os.getenv("PWD", "unknown")),
RuntimeWarning,
)
raise
return self._cwd
def set_default_config(self):
"""Read default settings template and set into config object"""
default_cfg = DEFAULT_CONFIG_TPL.format(
log_dir=os.path.expanduser(
"~"), # Get $HOME in a platform-agnostic way
crashdump_dir=self.cwd, # Read cached cwd
)
try:
self._config.read_string(default_cfg) # Python >= 3.2
except AttributeError:
from io import StringIO
self._config.readfp(StringIO(default_cfg))
def enable_debug_mode(self):
"""Enables debug configuration"""
from .. import logging
self._config.set("execution", "stop_on_first_crash", "true")
self._config.set("execution", "remove_unnecessary_outputs", "false")
self._config.set("execution", "keep_inputs", "true")
self._config.set("logging", "workflow_level", "DEBUG")
self._config.set("logging", "interface_level", "DEBUG")
self._config.set("logging", "utils_level", "DEBUG")
logging.update_logging(self._config)
def set_log_dir(self, log_dir):
"""Sets logging directory
This should be the first thing that is done before any nipype class
with logging is imported.
"""
self._config.set("logging", "log_directory", log_dir)
def get(self, section, option, default=None):
"""Get an option"""
if option in CONFIG_DEPRECATIONS:
msg = ('Config option "%s" has been deprecated as of nipype %s. '
'Please use "%s" instead.') % (
option, CONFIG_DEPRECATIONS[option][1],
CONFIG_DEPRECATIONS[option][0])
warn(msg)
section, option = CONFIG_DEPRECATIONS[option][0].split(".")
if self._config.has_option(section, option):
return self._config.get(section, option)
return default
def set(self, section, option, value):
"""Set new value on option"""
if isinstance(value, bool):
value = str(value)
if option in CONFIG_DEPRECATIONS:
msg = ('Config option "%s" has been deprecated as of nipype %s. '
'Please use "%s" instead.') % (
option, CONFIG_DEPRECATIONS[option][1],
CONFIG_DEPRECATIONS[option][0])
warn(msg)
section, option = CONFIG_DEPRECATIONS[option][0].split(".")
return self._config.set(section, option, value)
def getboolean(self, section, option):
"""Get a boolean option from section"""
return self._config.getboolean(section, option)
def has_option(self, section, option):
"""Check if option exists in section"""
return self._config.has_option(section, option)
@property
def _sections(self):
return self._config._sections
def get_data(self, key):
"""Read options file"""
if not os.path.exists(self.data_file):
return None
with SoftFileLock("%s.lock" % self.data_file):
with open(self.data_file, "rt") as file:
datadict = load(file)
if key in datadict:
return datadict[key]
return None
def save_data(self, key, value):
"""Store config flie"""
datadict = {}
if os.path.exists(self.data_file):
with SoftFileLock("%s.lock" % self.data_file):
with open(self.data_file, "rt") as file:
datadict = load(file)
else:
dirname = os.path.dirname(self.data_file)
if not os.path.exists(dirname):
mkdir_p(dirname)
with SoftFileLock("%s.lock" % self.data_file):
with open(self.data_file, "wt") as file:
datadict[key] = value
dump(datadict, file)
def update_config(self, config_dict):
"""Extend internal dictionary with config_dict"""
for section in ["execution", "logging", "check"]:
if section in config_dict:
for key, val in list(config_dict[section].items()):
if not key.startswith("__"):
self._config.set(section, key, str(val))
def update_matplotlib(self):
"""Set backend on matplotlib from options"""
import matplotlib
matplotlib.use(self.get("execution", "matplotlib_backend"))
def enable_provenance(self):
"""Sets provenance storing on"""
self._config.set("execution", "write_provenance", "true")
self._config.set("execution", "hash_method", "content")
@property
def resource_monitor(self):
"""Check if resource_monitor is available"""
if self._resource_monitor is not None:
return self._resource_monitor
# Cache config from nipype config
self.resource_monitor = (str2bool(
self._config.get("monitoring", "enabled")) or False)
return self._resource_monitor
@resource_monitor.setter
def resource_monitor(self, value):
# Accept string true/false values
if isinstance(value, (str, bytes)):
value = str2bool(value.lower())
if value is False:
self._resource_monitor = False
elif value is True:
if not self._resource_monitor:
# Before setting self._resource_monitor check psutil
# availability
self._resource_monitor = False
try:
import psutil
self._resource_monitor = LooseVersion(
psutil.__version__) >= LooseVersion("5.0")
except ImportError:
pass
finally:
if not self._resource_monitor:
warn("Could not enable the resource monitor: "
"psutil>=5.0 could not be imported.")
self._config.set("monitoring", "enabled",
("%s" % self._resource_monitor).lower())
def enable_resource_monitor(self):
"""Sets the resource monitor on"""
self.resource_monitor = True
def disable_resource_monitor(self):
"""Sets the resource monitor off"""
self.resource_monitor = False
def get_display(self):
"""Returns the first display available"""
# Check if an Xorg server is listening
# import subprocess as sp
# if not hasattr(sp, 'DEVNULL'):
# setattr(sp, 'DEVNULL', os.devnull)
# x_listening = bool(sp.call('ps au | grep -v grep | grep -i xorg',
# shell=True, stdout=sp.DEVNULL))
if self._display is not None:
return ":%d" % self._display.new_display
sysdisplay = None
if self._config.has_option("execution", "display_variable"):
sysdisplay = self._config.get("execution", "display_variable")
sysdisplay = sysdisplay or os.getenv("DISPLAY")
if sysdisplay:
from collections import namedtuple
def _mock():
pass
# Store a fake Xvfb object. Format - <host>:<display>[.<screen>]
ndisp = sysdisplay.split(":")[-1].split(".")[0]
Xvfb = namedtuple("Xvfb", ["new_display", "stop"])
self._display = Xvfb(int(ndisp), _mock)
return self.get_display()
else:
if "darwin" in sys.platform:
raise RuntimeError(
"Xvfb requires root permissions to run in OSX. Please "
"make sure that an X server is listening and set the "
"appropriate config on either $DISPLAY or nipype's "
'"display_variable" config. Valid X servers include '
"VNC, XQuartz, or manually started Xvfb.")
# If $DISPLAY is empty, it confuses Xvfb so unset
if sysdisplay == "":
del os.environ["DISPLAY"]
try:
from xvfbwrapper import Xvfb
except ImportError:
raise RuntimeError(
"A display server was required, but $DISPLAY is not "
"defined and Xvfb could not be imported.")
self._display = Xvfb(nolisten="tcp")
self._display.start()
# Older versions of xvfbwrapper used vdisplay_num
if not hasattr(self._display, "new_display"):
setattr(self._display, "new_display",
self._display.vdisplay_num)
return self.get_display()
def stop_display(self):
"""Closes the display if started"""
if self._display is not None:
from .. import logging
self._display.stop()
logging.getLogger("nipype.interface").debug(
"Closing display (if virtual)")
class Driver(threading.Thread):
def __init__(
self,
username,
password,
screenshot_path,
window_width=1920,
window_height=1080,
):
self.username = username
self.password = password
log_path = Config.bot_path + "log/" + username
if not os.path.exists(log_path):
os.makedirs(log_path)
self.interacting_users_path = log_path + "/interacting_users.pickle"
self.hashtags_path = log_path + "/hashtags.pickle"
self.action_list_path = log_path + "/action_list.pickle"
self.followed_users_all_time_path = log_path + "/followed_users_all_time.pickle"
self.accounts_to_unfollow_path = log_path + "/accounts_to_unfollow.pickle"
self.log_path = log_path + "/log.pickle"
self.running_path = log_path + "/running.pickle"
try:
with open(self.interacting_users_path, "rb") as f:
self.interacting_users = pickle.load(f)
except (FileNotFoundError, EOFError):
with open(self.interacting_users_path, "wb") as f:
self.interacting_users = []
pickle.dump([], f)
try:
with open(self.hashtags_path, "rb") as f:
self.hashtags = pickle.load(f)
except (FileNotFoundError, EOFError):
with open(self.hashtags_path, "wb") as f:
self.hashtags = {}
for h in Config.topics:
self.hashtags[h] = 2
pickle.dump(self.hashtags, f)
try:
with open(self.action_list_path, "rb") as f:
self.action_list = pickle.load(f)
except (FileNotFoundError, EOFError):
with open(self.action_list_path, "wb") as f:
self.action_list = {}
pickle.dump({}, f)
try:
with open(self.followed_users_all_time_path, "rb") as f:
self.followed_accounts = pickle.load(f)
except (FileNotFoundError, EOFError):
with open(self.followed_users_all_time_path, "wb") as f:
self.followed_accounts = {}
pickle.dump({}, f)
try:
with open(self.accounts_to_unfollow_path, "rb") as f:
self.accounts_to_unfollow = pickle.load(f)
except (FileNotFoundError, EOFError):
with open(self.accounts_to_unfollow_path, "wb") as f:
self.accounts_to_unfollow = []
pickle.dump([], f)
try:
from xvfbwrapper import Xvfb
try:
self.vdisplay = Xvfb()
self.vdisplay.start()
except EnvironmentError:
print(
"Selenium Webdriver will run without Xvfb. There was an error starting Xvfb."
)
except ImportError:
print(
"Selenium Webdriver will run without Xvfb. Install Xvfb to run Selenium Webdriver inside Xvfb."
)
path = os.path.dirname(os.path.realpath(__file__))
phantom_js_path = path + "/phantomjs"
self.browser = webdriver.PhantomJS(phantom_js_path)
self.browser.set_window_size(window_width, window_height)
self.screenshot_path = screenshot_path
self.dispatcher = Dispatcher(log_path=log_path)
super(Driver, self).__init__()
def start(self):
super(Driver, self).start()
def running(self):
try:
with open(self.running_path, "rb") as f:
return bool(pickle.load(f))
except (FileNotFoundError, EOFError):
return False
@staticmethod
def now():
return datetime.datetime.now()
def focus(self, element, browser):
if self.running():
browser.execute_script("arguments[0].focus();", element)
def user_followed_already(self, user):
if self.running():
return user in self.followed_accounts
def login(self, username, password, browser, log_path, timeout=5):
if self.running():
Log.update(self.screenshot_path, self.browser, log_path,
"Logging in")
browser.get(Config.start_url)
try:
username_field = WebDriverWait(browser, timeout).until(
ec.presence_of_element_located((By.NAME, "username")))
pass_field = WebDriverWait(browser, timeout).until(
ec.presence_of_element_located((By.NAME, "password")))
except TimeoutException:
Log.update(self.screenshot_path, self.browser, log_path,
'Timeout in login')
return
username_field.send_keys(username)
pass_field.send_keys(password)
pass_field.send_keys(Keys.RETURN)
Log.update(self.screenshot_path, self.browser, log_path,
"Logged in")
def update_action_list(self, author, action_type, topic):
if author not in self.action_list.keys():
value = {"type": action_type, "time": Driver.now(), "topic": topic}
self.action_list[author] = [value]
else:
value = {"type": action_type, "time": Driver.now(), "topic": topic}
author_actions = self.action_list[author]
author_actions.append(value)
self.action_list[author] = author_actions
with open(self.action_list_path, "wb") as f:
pickle.dump(self.action_list, f)
def on_dialog_page(self, browser, log_path, check_timeout=5):
if self.running():
return True
#try:
# WebDriverWait(browser, check_timeout).until(
# ec.presence_of_element_located((By.XPATH, Config.dialog_xpath))
# )
#except (TimeoutException, NoSuchElementException):
# Log.update(self.screenshot_path, self.browser, log_path, 'No longer on dialog page.')
# return False
#else:
# return True
def comment(self, topic, browser, log_path, timeout=5):
if self.running():
author = self.author(browser=browser, log_path=log_path)
query = Config.comments[randint(0, len(Config.comments) - 1)]
say = query.format(
author, Config.smileys[randint(0,
len(Config.smileys) - 1)])
try:
WebDriverWait(browser, timeout).until(
ec.presence_of_element_located(
(By.XPATH, Config.comment_xpath)))
WebDriverWait(browser, timeout).until(
ec.element_to_be_clickable(
(By.XPATH, Config.comment_xpath)))
comment_button = WebDriverWait(browser, timeout).until(
ec.element_to_be_clickable(
(By.XPATH, Config.comment_submit_xpath)))
except TimeoutException:
Log.update(self.screenshot_path, self.browser, log_path,
'Timeout in comment')
return
comment_button.click()
comment_field = browser.find_element_by_xpath(Config.comment_xpath)
comment_field.send_keys(say)
comment_field.send_keys(Keys.RETURN)
Log.update(
self.screenshot_path, self.browser, log_path,
"Commented on " + str(author) + "s picture with: " + say)
self.update_action_list(author=author,
action_type="comment",
topic=topic)
def search(self, browser, log_path, query):
if self.running():
browser.get("https://www.instagram.com/explore/tags/" + query +
"/")
Log.update(self.screenshot_path, self.browser, log_path,
"Searching for " + query + ".")
# Selects the first picture in a loaded topic screen
def select_first(self, browser, log_path, timeout=5):
if self.running():
try:
WebDriverWait(browser, timeout).until(
ec.presence_of_element_located(
(By.XPATH, Config.first_ele_xpath)))
pictures = browser.find_elements_by_xpath(
Config.first_ele_xpath)
except NoSuchElementException:
Log.update(
self.screenshot_path, self.browser, log_path,
'NoSuchElementException in select_first: ' +
str(format_exc()))
return
except TimeoutException:
Log.update(self.screenshot_path, self.browser, log_path,
'Timeout in select_first')
return
if len(pictures) > 9:
first_picture = pictures[9]
else:
first_picture = pictures[len(pictures) - 1]
self.focus(first_picture, browser=browser)
first_picture.click()
def next_picture(self, browser):
if self.running():
actions = ActionChains(browser)
actions.send_keys(Keys.ARROW_RIGHT)
actions.perform()
def author(self, browser, log_path, timeout=5):
if self.running():
try:
author_element = WebDriverWait(browser, timeout).until(
ec.presence_of_element_located(
(By.XPATH, Config.author_xpath)))
except TimeoutException:
Log.update(self.screenshot_path, self.browser, log_path,
'Timeout in author')
return
return str(author_element.get_attribute("title"))
def already_liked(self, browser, log_path, error_timeout=5):
if self.running():
try:
WebDriverWait(browser, error_timeout).until(
ec.presence_of_element_located(
(By.XPATH, Config.like_button_full_xpath)))
except TimeoutException:
return False
Log.update(self.screenshot_path, self.browser, log_path,
'Post was already liked.')
return True
# Likes a picture
def like(self, browser, log_path, topic, timeout=5):
if self.running():
author = self.author(browser=browser, log_path=log_path)
try:
WebDriverWait(browser, timeout).until(
ec.presence_of_element_located(
(By.XPATH, Config.like_button_xpath)))
like_button = WebDriverWait(browser, timeout).until(
ec.element_to_be_clickable(
(By.XPATH, Config.like_button_xpath)))
except TimeoutException:
Log.update(self.screenshot_path, self.browser, log_path,
'Timeout in like')
return
like_button.click()
src = self.extract_picture_source(browser=browser,
log_path=log_path)
Log.update(self.screenshot_path,
self.browser,
log_path,
"Liked picture/video by: " + author,
image=src)
self.update_action_list(author=author,
action_type="like",
topic=topic)
# Unfollows a user
def unfollow(self, browser, log_path, name, timeout=5):
if self.running():
browser.get("https://www.instagram.com/" + name + "/")
try:
WebDriverWait(browser, timeout).until(
ec.presence_of_element_located(
(By.XPATH, Config.unfollow_xpath)))
unfollow_button = WebDriverWait(browser, timeout).until(
ec.element_to_be_clickable(
(By.XPATH, Config.unfollow_xpath)))
except TimeoutException:
Log.update(self.screenshot_path, self.browser, log_path,
'Timeout in unfollow')
return
unfollow_button.click()
Log.update(self.screenshot_path, self.browser, log_path,
"Unfollowed: " + name)
def update_accounts_to_unfollow(self, author):
self.accounts_to_unfollow.append(author)
with open(self.accounts_to_unfollow_path, "wb") as f:
pickle.dump(self.accounts_to_unfollow, f)
def update_followed_accounts(self, author):
self.followed_accounts.update({author: Driver.now()})
with open(self.followed_users_all_time_path, "wb") as userfile:
pickle.dump(self.followed_accounts, userfile)
# Follows a user
def follow(self, browser, log_path, topic, timeout=15):
if self.running():
author = self.author(browser=browser, log_path=log_path)
try:
WebDriverWait(browser, timeout).until(
ec.presence_of_element_located(
(By.XPATH, Config.follow_xpath)))
follow_button = WebDriverWait(browser, timeout).until(
ec.element_to_be_clickable(
(By.XPATH, Config.follow_xpath)))
except TimeoutException:
Log.update(self.screenshot_path, self.browser, log_path,
'Timeout in follow')
return
follow_button.click()
Log.update(self.screenshot_path,
browser=self.browser,
log_path=self.log_path,
text="Followed: " + author)
self.update_action_list(author=author,
action_type="follow",
topic=topic)
self.update_accounts_to_unfollow(author=author)
self.update_followed_accounts(author=author)
def open_unfollow_screen(self, browser, log_path, timeout=15):
if self.running():
try:
WebDriverWait(browser, timeout).until(
ec.presence_of_element_located(
(By.XPATH, Config.following_xpath)))
heart = WebDriverWait(browser, timeout).until(
ec.element_to_be_clickable(
(By.XPATH, Config.following_xpath)))
except TimeoutException:
Log.update(self.screenshot_path, self.browser, log_path,
'Timeout in open_unfollow_screen')
return
heart.click()
def update_interacting_users(self, user):
self.interacting_users.append(user)
with open(self.interacting_users_path, "wb") as f:
pickle.dump(self.interacting_users, f)
def check_follows(self, browser, log_path, timeout=15):
if self.running():
try:
WebDriverWait(browser, timeout).until(
ec.presence_of_element_located(
(By.XPATH, Config.sections_xpath)))
sections = browser.find_elements_by_xpath(
Config.sections_xpath)
except NoSuchElementException:
Log.update(
self.screenshot_path, self.browser, log_path,
'NoSuchElementException in check_follows: ' +
str(format_exc()))
return
except TimeoutException:
Log.update(self.screenshot_path, self.browser, log_path,
'Timeout in check_follows')
return
users = []
for element in sections:
try:
profile = element.find_element_by_xpath(
Config.local_name_xpath)
except NoSuchElementException:
Log.update(
self.screenshot_path, self.browser, log_path,
'NoSuchElementException in check_follows: ' +
str(format_exc()))
return
name = profile.get_attribute("title")
users.append(name)
for user in users:
if user not in self.interacting_users:
if user in self.action_list.keys():
actions = self.action_list[user]
for action in actions:
self.hashtags[action["topic"]] += 1
self.update_interacting_users(user=user)
def update_hashtags(self, hashtag, boost=0.1):
if hashtag in self.hashtags:
self.hashtags[hashtag] += boost
else:
self.hashtags[hashtag] = boost
with open(self.hashtags_path, "wb") as f:
pickle.dump(self.hashtags, f)
def store_hashtags(self, browser, log_path, timeout=5):
if self.running():
all_hashtags = self.extract_hash_tags(browser=browser,
log_path=log_path,
timeout=timeout)
for hashtag in all_hashtags:
self.update_hashtags(hashtag=hashtag)
def extract_hash_tags(self, browser, log_path, timeout=5):
if self.running():
try:
WebDriverWait(browser, timeout).until(
ec.presence_of_element_located(
(By.XPATH, Config.hashtags_xpath)))
sections = browser.find_elements_by_xpath(
Config.hashtags_xpath)
except NoSuchElementException:
Log.update(
self.screenshot_path, self.browser, log_path,
'Exception in extract_hash_tags: ' + str(format_exc()))
return []
except TimeoutException:
Log.update(self.screenshot_path, self.browser, log_path,
'Timeout in extract_hash_tags')
return []
all_hashtags = []
for section in sections:
all_hashtags.extend(
set(part[1:] for part in section.text.split()
if part.startswith('#')))
return all_hashtags
def extract_picture_source(self, browser, log_path, timeout=5):
if self.running():
try:
WebDriverWait(browser, timeout).until(
ec.presence_of_element_located(
(By.XPATH, Config.image_div_container_xpath)))
sections = browser.find_elements_by_xpath(
Config.image_div_container_xpath)
except NoSuchElementException:
Log.update(
self.screenshot_path, self.browser, log_path,
'Exception in extract_picture_source: ' +
str(format_exc()))
return
except TimeoutException:
Log.update(self.screenshot_path, self.browser, log_path,
'Timeout in extract_picture_source')
return
for section in sections:
try:
image = section.find_element_by_tag_name("img")
except NoSuchElementException:
Log.update(
self.screenshot_path, self.browser, log_path,
'Exception in extract_picture_source: ' +
str(format_exc()))
return
return image.get_attribute("src")
def post_hashtags_are_sfw(self, browser, log_path, timeout=5):
if self.running():
all_hashtags = [
"#" + x for x in self.extract_hash_tags(
browser=browser, log_path=log_path, timeout=timeout)
]
for hashtag in all_hashtags:
if hashtag in Config.nsfw_hashtags:
Log.update(
screenshot_path=self.screenshot_path,
browser=self.browser,
log_path=self.log_path,
text="This post contains the blacklisted hashtag {}.".
format(hashtag),
)
return False
Log.update(
screenshot_path=self.screenshot_path,
browser=self.browser,
log_path=self.log_path,
text=
"This post contains none of the blacklisted hashtags. (Hashtags: {})"
.format(", ".join(all_hashtags)),
)
return True
def post_is_sfw(self, browser, log_path, limit=0.1):
if self.running():
if not self.post_hashtags_are_sfw(browser=browser,
log_path=log_path):
return False
image_url = self.extract_picture_source(browser=browser,
log_path=log_path)
if not image_url:
Log.update(screenshot_path=self.screenshot_path,
browser=self.browser,
log_path=self.log_path,
text="Picture source could not be extracted.")
return True
sfw, nsfw = classify_nsfw(image_url)
Log.update(
screenshot_path=self.screenshot_path,
browser=self.browser,
log_path=self.log_path,
text="Analysis of this post yielded it to be {}% SFW.".format(
int(100 * sfw)),
image=image_url)
return nsfw < limit
def run(self):
self.login(browser=self.browser,
log_path=self.log_path,
password=self.password,
username=self.username)
while self.running():
try:
self.open_unfollow_screen(browser=self.browser,
log_path=self.log_path)
self.check_follows(browser=self.browser,
log_path=self.log_path)
top_hashtags = sorted(self.hashtags.keys(),
key=lambda k: self.hashtags[k],
reverse=True)[:20]
for i, topic in enumerate(top_hashtags):
self.search(query=topic,
browser=self.browser,
log_path=self.log_path)
self.select_first(browser=self.browser,
log_path=self.log_path)
delay, action = self.dispatcher.next_action()
Log.update(
self.screenshot_path, self.browser, self.log_path,
"Dispatcher selected action: {} (Sleeping {}s)".format(
action, delay))
sleep(delay)
if action == "comment":
if self.post_is_sfw(browser=self.browser,
log_path=self.log_path):
self.comment(topic=topic,
browser=self.browser,
log_path=self.log_path)
self.dispatcher.log_action("comment")
self.store_hashtags(browser=self.browser,
log_path=self.log_path)
elif action == "like":
count = 0
while self.already_liked(browser=self.browser,
log_path=self.log_path):
if not self.on_dialog_page(self.browser,
self.log_path):
break
if count > 10:
break
self.next_picture(browser=self.browser)
count += 1
if self.on_dialog_page(self.browser, self.log_path):
if self.post_is_sfw(browser=self.browser,
log_path=self.log_path):
self.like(topic=topic,
browser=self.browser,
log_path=self.log_path)
self.dispatcher.log_action("like")
self.store_hashtags(browser=self.browser,
log_path=self.log_path)
elif action == "follow":
count = 0
while self.user_followed_already(
self.author(browser=self.browser,
log_path=self.log_path)):
if not self.on_dialog_page(self.browser,
self.log_path):
break
if count > 10:
break
self.next_picture(browser=self.browser)
count += 1
if self.on_dialog_page(self.browser, self.log_path):
if self.post_is_sfw(browser=self.browser,
log_path=self.log_path):
self.follow(topic=topic,
browser=self.browser,
log_path=self.log_path)
self.dispatcher.log_action("follow")
self.store_hashtags(browser=self.browser,
log_path=self.log_path)
elif action == "unfollow":
if len(self.accounts_to_unfollow) > 50:
this_guy = self.accounts_to_unfollow[0]
self.unfollow(name=this_guy,
browser=self.browser,
log_path=self.log_path)
del self.accounts_to_unfollow[0]
self.dispatcher.log_action("unfollow")
except Exception:
Log.update(self.screenshot_path,
self.browser,
self.log_path,
text='General Exception: ' + str(format_exc()))
Log.update(self.screenshot_path,
self.browser,
self.log_path,
text='Stopped bot')
if self.vdisplay:
self.vdisplay.stop()
super(Driver, self).join()
class Surfer:
"""This class does virtual web surfing on our demand
"""
def __init__(self, delay=1, browser="firefox"):
"""delay: Number of extra seconds to wait when a page is
supposedly loaded. Try raising this in case of weird errors.
browser: `firefox` or `chrome`. The ChromeDriver executable for your
OS must be inside the bin directory for Chrome to work. Get it from:
http://chromedriver.storage.googleapis.com/index.html
"""
self.extra_delay = delay # extra time to wait after each operation (s)
self.temp_dir = mkdtemp()
self.vdisplay = Xvfb()
self.vdisplay.start()
def get_bin_path(subdir):
path = os_path.dirname(os_path.realpath(__file__)) + os_sep
return path + os_sep.join([pardir, "bin", subdir])
if browser == "firefox":
profile = FirefoxProfile()
# Open links in same window
profile.set_preference("browser.link.open_newwindow", 1)
# Download to temp dir, for files we can't open inline
profile.set_preference("browser.download.dir", self.temp_dir)
profile.set_preference("browser.download.folderList", 2)
profile.set_preference("browser.download.manager.showWhenStarting",
"False")
profile.set_preference(
"browser.helperApps.neverAsk.saveToDisk",
"application/msword, application/vnd.ms-word, application/rtf, application/octet-stream"
)
# Add extension for overriding Content-Disposition headers, etc
extensions_dir = get_bin_path('firefox-plugins-enabled')
for filename in listdir(extensions_dir):
fullfilename = os_sep.join([extensions_dir, filename])
profile.add_extension(extension=fullfilename)
driver = Firefox(profile)
elif browser == "chrome":
# Add extension for overriding Content-Disposition headers
options = ChromeOptions()
options.add_extension(get_bin_path('bin/undisposition.crx'))
driver = Chrome(executable_path=get_bin_path('bin/chromedriver'),
chrome_options=options)
else:
raise Exception("Not a valid browser name")
self.selenium_driver = EventFiringWebDriver(driver, CustomListener())
"""selenium_driver is a EventFiringWebDriver, so that it can
trigger javascript event
"""
self.browser_version = " ".join([
self.selenium_driver.capabilities['browserName'],
self.selenium_driver.capabilities['version']
]) # 'Firefox 33.0'
def get_last_download(self):
files = sorted([f for f in listdir(self.temp_dir)])
if len(files) > 0:
return self.temp_dir + os_sep + files[-1]
else:
return None
def _get_nearest_ancestor(self, element, tagname):
ancestor = element.find_element_by_xpath("..")
while ancestor is not None and ancestor.tag_name != tagname:
try:
ancestor = element.find_element_by_xpath("..")
except InvalidSelectorException:
ancestor = None
return ancestor
def _scroll_element_into_view(self, element):
"""Scroll attached element into view
"""
y = element.location['y']
self.selenium_driver.execute_script(
'window.scrollTo(0, {0})'.format(y))
def with_open_in_new_window(self, element, callback_, *args, **kwargs):
"""Shift-clicks on an element to open a new window,
calls the callback function, and then closes the
window and returns.
This is useful for iterating through a link tree
we cannot easily step back in (e.g. where the starting
the result of a POST request)
Callback function is called like this: `callback(Surfer, arguments)`
with a Surfer object placed in the new window
Returns the result of `callback`
"""
try:
actions = ActionChains(self.selenium_driver)
actions.move_to_element(element).perform()
self._scroll_element_into_view(element)
element.send_keys(Keys.SHIFT + Keys.ENTER)
except ElementNotVisibleException:
raise
self.selenium_driver.implicitly_wait(self.extra_delay)
sleep(self.extra_delay) # implicitly_wait doesn't work on FF?
windows = self.selenium_driver.window_handles
self.selenium_driver.switch_to_window(windows[-1])
if len(windows) > 1:
res = callback_(self, *args, **kwargs)
self.selenium_driver.close()
windows = self.selenium_driver.window_handles
self.selenium_driver.switch_to_window(windows[-1])
return res
else:
return None
def surf_to(self, url):
"""Simply go to an URL"""
try:
self.selenium_driver.get(url)
except TimeoutException:
raise ConnectionError
except BadStatusLine:
raise ConnectionError
self.selenium_driver.implicitly_wait(self.extra_delay)
def click_on_stuff(self, xpath):
"""Will click on any elements pointed out by xPath.
Options in select menus will be selected, and an
onChange event fired (as this does not always happen automatically)
"""
# FIXME: Select menus will only work if they have an id!
element_list = self.selenium_driver.find_elements_by_xpath(xpath)
if not element_list:
raise Exception("No elements found for xPath `%s`" % xpath)
else:
for element in element_list:
try:
element.click()
except ElementNotVisibleException:
# Element not visible. This is not necessarily an error.
# but the user might want to consider using a more
# specific xPath expression.
continue
# actions = ActionChains(self.selenium_driver)
# actions.move_to_element(element)
# actions.click(element)
# actions.send_keys_to_element(element, Keys.ENTER)
# actions.perform()
if "tag_name" in dir(element) and element.tag_name == "option":
parent = self._get_nearest_ancestor(element, "select")
if parent is not None:
# Should be selected already, when clicking,
# but it seems like this does not always happen
value = element.get_attribute("value") or None
if value is not None:
select = Select(parent)
select.select_by_value(value)
# Manually trigger JS events
select_id = parent.get_attribute("id") or None
if select_id is not None:
js_function = """
window.triggerChange=function(){\
var el = document.getElementById('""" + select_id + """');\
el.dispatchEvent(new Event('change', { 'bubbles': true }));\
el.dispatchEvent(new Event('select', { 'bubbles': true }));\
el.dispatchEvent(new Event('click', { 'bubbles': true }));\
};"""
self.selenium_driver.execute_script(js_function)
self.selenium_driver.execute_script(
"triggerChange();")
else:
raise Exception("No <select> tag found for <option>")
self.selenium_driver.implicitly_wait(self.extra_delay)
def get_url_list(self, xpath):
url_list = []
element_list = self.selenium_driver.find_elements_by_xpath(xpath)
for element in element_list:
href = element.get_attribute("href")
if href is not None:
url_list.append(Url(href))
return url_list # list of Url objects
def get_element_list(self, xpath):
try:
return self.selenium_driver.find_elements_by_xpath(xpath)
except InvalidSelectorException:
pass
# maybe our xPath points at an attribute?
try:
return self.selenium_driver.find_elements_by_xpath(xpath + "/..")
except InvalidSelectorException:
pass
return None
def kill(self):
self.selenium_driver.close()
self.vdisplay.stop()
rmtree(self.temp_dir)
def test_xvfb(self):
vdisplay = Xvfb()
vdisplay.start()
display_var = ':{}'.format(vdisplay.new_display)
self.assertEqual(display_var, os.environ['DISPLAY'])
vdisplay.stop()
def xvfb_launcher(request):
if config.browser.headless:
vdisplay = Xvfb()
vdisplay.start()
return vdisplay
class ParseSettleDataFromCme:
CME_LINK = 'https://www.cmegroup.com'
CME_TOOLS_LINK = 'https://cmegroup-tools.quikstrike.net'
MAIN_URL = CME_LINK + '/tools-information/quikstrike/option-settlement.html'
STRIKES_URL = CME_TOOLS_LINK + '/User/QuikStrikeView.aspx?pid={pid}&pf=61&viewitemid=IntegratedSettlementSheet'
OPTION_TAB_ID = 'ctl00_ucSelector_lvGroups_ctrl{option_index}_lbGroup'
OPTION_GROUP_ID = 'ctl00_ucSelector_lvGroups_ctrl{option_index}_ctl00_pnlExpirationGroupPopup'
OPTION_UL_CLASS = 'nav'
OPTION_SPAN_CLASS = 'item-name'
ROW_QUANTITY_ID = 'MainContent_ucViewControl_IntegratedSettlementSheet_ddlStrikeCount'
ROW_QUANTITY_VALUE = '50'
MONTHLY_OPTION = 'Monthly'
FRIDAY_OPTION = 'Friday'
WEDNESDAY_OPTION = 'Wednesday'
MONTHLY_INDEX = 0
FRIDAY_INDEX = 2
WEDNESDAY_INDEX = 4
def __init__(self, option):
self.vdisplay = Xvfb()
self.driver = None
self.symbol = option.symbol
self.pid = option.symbol.cme_pid
self.cab = option.symbol.cab
self.option_type = option.option_type
self.option_code = option.option_code
def _display_start(self):
self._display_stop()
try:
self.vdisplay.start()
except:
print('Start without virtual display')
def _display_stop(self):
try:
self.vdisplay.stop()
except:
print('Virtual display can not stoped')
def _webdriver_start(self):
driver = webdriver.Firefox(executable_path=GECKODRIVER_PATH,
log_path=GECKODRIVER_LOG_PATH)
driver.maximize_window()
driver.implicitly_wait(15)
self.driver = driver
def _webdriver_stop(self):
try:
self.driver.quit()
except WebDriverException:
print('Ceckodriver can not stoped')
def open(self):
self._display_start()
self._webdriver_start()
def close(self):
self._webdriver_stop()
self._display_stop()
def parse(self):
url = self.STRIKES_URL.format(pid=self.pid)
option_index = 0
self.driver.get(self.MAIN_URL)
sleep(5)
self.driver.get(url)
sleep(5)
if self.option_type == self.MONTHLY_OPTION:
option_index = self.MONTHLY_INDEX
elif self.option_type == self.FRIDAY_OPTION:
option_index = self.FRIDAY_INDEX
if self.symbol.symbol != "JPYUSD":
raw_code = self.option_code
self.option_code = raw_code[1] + raw_code[2] + raw_code[
0] + raw_code[3] + raw_code[4]
elif self.option_type == self.WEDNESDAY_OPTION:
option_index = self.WEDNESDAY_INDEX
self.driver.find_element_by_xpath('//a[@id="' +
self.OPTION_TAB_ID.format(
option_index=option_index) +
'"]').click()
sleep(5)
soup = BeautifulSoup(self.driver.page_source, 'html.parser')
div_list = soup.find(
'div',
{'id': self.OPTION_GROUP_ID.format(option_index=option_index)})
li_list = div_list.find('ul', {
'class': self.OPTION_UL_CLASS
}).find_all('li')
for li in li_list:
if li.find('span', {
'class': self.OPTION_SPAN_CLASS
}).text == self.option_code:
put_strike = 0
call_strike = 0
link_id = li.find('a').attrs['id']
self.driver.find_element_by_xpath('//a[@id="' + link_id +
'"]').click()
self.driver.find_element_by_xpath('//select[@id="' +
self.ROW_QUANTITY_ID +
'"]/option[@value="' +
self.ROW_QUANTITY_VALUE +
'"]').click()
sleep(5)
soup = BeautifulSoup(self.driver.page_source, 'html.parser')
strikes_data = []
table = soup.find('table', {'id': 'pricing-sheet'})
for row in table.tbody.find_all('tr'):
tds = row.find_all('td')
tds = [
td for td in tds
if 'hide' not in td.attrs.get('class', [])
]
strike_items = [
utils.str_to_numbers(tds[2].text),
utils.str_to_numbers(tds[3].text),
utils.str_to_numbers(tds[4].text)
]
strikes_data.append(strike_items)
for strike_item in strikes_data:
if strike_item[0] == self.cab:
call_strike = strike_item[1]
break
else:
continue
reverse_strikes_data = sorted(strikes_data,
key=lambda item: item[1],
reverse=True)
for strike_item in reverse_strikes_data:
if strike_item[2] == self.cab:
put_strike = strike_item[1]
break
else:
continue
balance = round((call_strike + put_strike) / 2, 5)
cab_data = {
"call_strike": call_strike,
"balance": balance,
"put_strike": put_strike,
}
return cab_data
utils.print_error("Contract not found.")
return 0
def create_snippet(approach,
url,
screen_size=None,
driver=None,
browser="chrome",
options=None,
output=None,
prettyfy=True,
css_selector=None,
xpath_selector=None,
path_to_js_script=None,
xvfb=False,
prettyfy_parser="html.parser",
wait=None):
if approach == COMPUTE_CSS:
filename_of_snippet_script = "create_snippet_using_computed_css.js"
elif approach == EXTRACT_CSS:
filename_of_snippet_script = "create_snippet_extracting_css_styles.js"
else:
raise Exception("Unknown approach")
path_to_snippet_script = os.path.abspath(
os.path.join(os.path.dirname(__file__), filename_of_snippet_script))
if xvfb:
try:
from xvfbwrapper import Xvfb
except ImportError:
raise Exception(
"xvfbwrapper is required. Install the module with this command: pip install xvfbwrapper"
)
vdisplay = Xvfb()
vdisplay.start()
if driver is None:
from selenium import webdriver
if browser == "chrome":
driver = webdriver.Chrome()
elif browser == "firefox":
driver = webdriver.Firefox()
elif browser == "edge":
driver = webdriver.Edge()
elif browser == "phantomjs":
driver = webdriver.PhantomJS()
else:
raise Exception("Browser not valid: %s" % browser)
if screen_size is None:
driver.maximize_window()
else:
driver.set_window_size(*screen_size)
if approach == COMPUTE_CSS and options is not None and options.get(
"removeDefaultCssValues", False):
path_to_empty_html = local_to_absolute_path("empty.html")
empty_url = path2url(path_to_empty_html)
with wait_for_page_load(driver):
driver.get(empty_url)
path_to_default_values_script = local_to_absolute_path(
"default_browser_css_values.js")
with open(path_to_default_values_script, "r") as f:
default_browser_css_values_js = f.read()
default_browser_css_values_js += "\nreturn default_browser_css_values()"
default_values = driver.execute_script(default_browser_css_values_js)
options["defaultValues"] = default_values
root = None
with wait_for_page_load(driver):
driver.get(url)
if wait is not None:
time.sleep(float(wait))
if path_to_js_script is not None:
from selenium.webdriver.remote.webelement import WebElement
with open(path_to_js_script) as f:
js_script = f.read()
root = driver.execute_script(js_script)
if not isinstance(root, WebElement):
root = None
if root is None:
if css_selector is not None:
root = driver.find_element_by_css_selector(css_selector)
elif xpath_selector is not None:
root = driver.find_element_by_xpath(xpath_selector)
else:
raise Exception("No selector indicated")
with open(path_to_snippet_script, "r") as f:
snippet_script = f.read()
snippet_script += "\nreturn createSnippet(arguments[0], arguments[1])"
snippet = driver.execute_script(snippet_script, root, options)
if prettyfy:
from bs4 import BeautifulSoup
snippet = BeautifulSoup(snippet,
prettyfy_parser).prettify(encoding='utf-8')
else:
snippet = snippet.encode("utf-8")
if xvfb:
vdisplay.stop()
return snippet
class TestHitAnalysis(unittest.TestCase):
@classmethod
def setUpClass(self):
if os.getenv('TRAVIS', False):
from xvfbwrapper import Xvfb # virtual X server for plots under headless LINUX travis testing is needed
self.vdisplay = Xvfb()
self.vdisplay.start()
self.simulate_data = simulate_data.SimulateData(random_seed=0)
@classmethod
def tearDownClass(self): # remove created files
for dut_index in range(self.simulate_data.n_duts):
os.remove('simulated_data_DUT%d.h5' % dut_index)
def test_position(
self): # Test beam position with respect to devices positions
self.simulate_data.n_duts = 2
self.simulate_data.set_std_settings()
def check_position(
): # Helper function to be called with different position parameter data
# Calculate expectation
expected_mean_column, expected_mean_row = [], []
for dut_index in range(self.simulate_data.n_duts):
expected_mean_column.append(
self.simulate_data.beam_position[0] -
self.simulate_data.offsets[dut_index][0])
expected_mean_row.append(
self.simulate_data.beam_position[1] -
self.simulate_data.offsets[dut_index][1])
# Extract results
mean_column, mean_row = [], []
for dut_index in range(self.simulate_data.n_duts):
with tb.open_file('simulated_data_DUT%d.h5' % dut_index,
'r') as in_file_h5:
mean_column.append(
(in_file_h5.root.Hits[:]['column'].mean() - 1) *
self.simulate_data.dut_pixel_size[dut_index][0])
mean_row.append(
(in_file_h5.root.Hits[:]['row'].mean() - 1) *
self.simulate_data.dut_pixel_size[dut_index][1])
self.assertTrue(
np.allclose(expected_mean_column,
mean_column,
rtol=0.01,
atol=10))
self.assertTrue(
np.allclose(expected_mean_row, mean_row, rtol=0.01, atol=10))
# Test 1: Check different DUT offsets
self.simulate_data.offsets = [(-35000, -35000), (-30000, -30000),
(-25000, -25000), (-20000, -20000),
(-15000, -15000), (-10000, -10000)
] # Set DUT offsets with respect to beam
self.simulate_data.create_data_and_store('simulated_data',
n_events=10000)
check_position()
# Test 2: Check different beam offset
self.simulate_data.beam_position = (500, 500) # Shift beam position
self.simulate_data.create_data_and_store('simulated_data',
n_events=10000)
check_position()
# Test 3: Check different beam parameter
self.simulate_data.beam_position_sigma = (0, 0) # beam position sigma
self.simulate_data.beam_position = (0, 0) # Shift beam position
self.simulate_data.create_data_and_store('simulated_data',
n_events=10000)
check_position()
def test_charge_distribution(self):
self.simulate_data.reset()
def check_charge(
): # Helper function to be called with different charge parameter data
for dut_index in range(
self.simulate_data.n_duts): # Loop over DUTs
mpv_charge = 71. * self.simulate_data.dut_thickness[
dut_index] # 71 electrons per um in silicon
def _calc_landau(x):
if self.simulate_data.dut_noise[dut_index]:
# PyLandau Langau MPV parameter is the MPV of the Langau not the Landau only!
y = pylandau.langau(
x,
mpv=mpv_charge,
eta=mpv_charge / 10.,
sigma=self.simulate_data.dut_noise[dut_index],
scale_langau=False)
else:
y = pylandau.landau(x,
mpv=mpv_charge,
eta=mpv_charge / 10)
return y
# Check result with calculated expectation
with tb.open_file('simulated_data_DUT%d.h5' % dut_index,
'r') as in_file_h5:
charge = in_file_h5.root.Hits[:][
'charge'] * 10. # 1 LSB corresponds to 10 electrons
charge_hist, edges = np.histogram(charge,
bins=100,
range=(0.,
5. * mpv_charge))
charge_hist = charge_hist.astype(np.float)
x = (edges[:-1] + edges[1:]) / 2
y = _calc_landau(x)
self.assertTrue(
np.allclose(y / y.sum() * charge_hist.sum(),
charge_hist,
rtol=0.07,
atol=50.))
# import matplotlib.pyplot as plt
# plt.plot(x, charge_hist, label='Result')
# plt.plot(x, y / y.sum() * charge_hist.sum(), label='Expected')
# print x[y == np.amax(y)]
# print x[charge_hist == np.amax(charge_hist)]
# plt.legend(loc=0)
# plt.show()
# Check Landau for different device thickness
self.simulate_data.dut_thickness = [
(i + 1) * 100 for i in range(self.simulate_data.n_duts)
] # Create charge distribution in different device thicknesses, thus Landau MPW should change
self.simulate_data.digitization_charge_sharing = False # To judge deposited charge, charge sharing has to be off
self.simulate_data.create_data_and_store('simulated_data',
n_events=100000)
check_charge()
# Check Landau for different device noiose
self.simulate_data.reset()
self.simulate_data.dut_thickness = [
200 for i in range(self.simulate_data.n_duts)
] # Fix device thickness
self.simulate_data.dut_noise = [
i * 1000 for i in range(self.simulate_data.n_duts)
] # Create charge distribution in different device noise, thus Langau sigma should change
self.simulate_data.digitization_charge_sharing = False # To judge deposited charge, charge sharing has to be off
self.simulate_data.create_data_and_store('simulated_data',
n_events=100000)
check_charge()
@unittest.SkipTest # FIXME: FAILS
def test_beam_angle(self):
self.simulate_data.reset()
def check_beam_angle():
# Expected offsets in x, y at DUT planes due to initial beam angle theta and direction distribution phi = (start, stop)
expected_offsets_x, expected_offsets_y = [], []
if self.simulate_data.beam_direction[
0] < self.simulate_data.beam_direction[1]:
mean_direction_cos = np.cos(
np.arange(self.simulate_data.beam_direction[0],
self.simulate_data.beam_direction[1], 0.01)
).mean(
) # A mean angle does not translate linearly to a mean offset
mean_direction_sin = np.sin(
np.arange(self.simulate_data.beam_direction[0],
self.simulate_data.beam_direction[1],
0.01)).mean()
else:
mean_direction_cos = np.cos(
self.simulate_data.beam_direction[0])
mean_direction_sin = np.sin(
self.simulate_data.beam_direction[0])
for dut_index in range(self.simulate_data.n_duts):
offset = self.simulate_data.beam_position[
0] - self.simulate_data.offsets[dut_index][0]
expected_offsets_x.append(
offset + mean_direction_cos *
np.tan(self.simulate_data.beam_angle / 1000.) *
self.simulate_data.z_positions[dut_index])
expected_offsets_y.append(
offset + mean_direction_sin *
np.tan(self.simulate_data.beam_angle / 1000.) *
self.simulate_data.z_positions[dut_index])
# Extract results
mean_column, mean_row = [], []
for dut_index in range(self.simulate_data.n_duts):
with tb.open_file('simulated_data_DUT%d.h5' % dut_index,
'r') as in_file_h5:
mean_column.append(
(in_file_h5.root.Hits[:]['column'].mean() - 1) *
self.simulate_data.dut_pixel_size[dut_index][0])
mean_row.append(
(in_file_h5.root.Hits[:]['row'].mean() - 1) *
self.simulate_data.dut_pixel_size[dut_index][1])
# Check for similarity, on pixel width error expected (binning error)
self.assertTrue(
np.allclose(expected_offsets_x,
mean_column,
rtol=0.001,
atol=self.simulate_data.dut_pixel_size[0][0]))
self.assertTrue(
np.allclose(expected_offsets_y,
mean_row,
rtol=0.001,
atol=self.simulate_data.dut_pixel_size[0][0]))
# Test 1: Fixed theta angle, different fixed phi
self.simulate_data.beam_angle = 5 # If the angle is too small this tests fails due to pixel discretisation error
self.simulate_data.dut_pixel_size = [
(1, 1)
] * self.simulate_data.n_duts # If the pixel size is too big this tests fails due to pixel discretisation error
self.simulate_data.dut_n_pixel = [
(10000, 10000)
] * self.simulate_data.n_duts # If the sensor is too small the mean cannot be easily calculated
self.simulate_data.beam_angle_sigma = 0
self.simulate_data.beam_position_sigma = (0, 0)
self.simulate_data.dut_material_budget = [
0
] * self.simulate_data.n_duts # Turn off multiple scattering
self.simulate_data.digitization_charge_sharing = False # Simplify position reconstruction
self.simulate_data.shuffle_hits = False
for phi in [
0, np.pi / 4., np.pi / 2., 3. / 4. * np.pi, np.pi,
5. * np.pi / 4., 3 * np.pi / 2.
]:
self.simulate_data.beam_direction = (phi, phi)
self.simulate_data.create_data_and_store('simulated_data',
n_events=10000)
check_beam_angle()
# Test 2: Fixed theta angle, different phi ranges
for phi_max in [
0, np.pi / 4., np.pi / 2., 3. / 4. * np.pi, np.pi,
5. * np.pi / 4., 3 * np.pi / 2.
]:
self.simulate_data.beam_direction = (0, phi_max)
self.simulate_data.create_data_and_store('simulated_data',
n_events=10000)
check_beam_angle()
# Test 3: Fixed theta angle, different phi ranges
for phi_max in [
0, np.pi / 4., np.pi / 2., 3. / 4. * np.pi, np.pi,
5. * np.pi / 4., 3 * np.pi / 2.
]:
self.simulate_data.beam_direction = (0, phi_max)
self.simulate_data.create_data_and_store('simulated_data',
n_events=10000)
check_beam_angle()
# Test 4: Gaussian dstributed theta angle, full phi range
self.simulate_data.beam_angle_sigma = 2
self.simulate_data.beam_direction = (0, 2. * np.pi)
self.simulate_data.create_data_and_store('simulated_data',
n_events=10000)
check_beam_angle()
def test_multiple_scattering(self):
self.simulate_data.reset()
# Set two planes and check the scattering angle due to the material budget of the first plane
self.simulate_data.n_duts = 2
self.simulate_data.set_std_settings()
self.simulate_data.beam_angle = 0
self.simulate_data.beam_angle_sigma = 0
self.simulate_data.beam_position_sigma = (0, 0)
self.simulate_data.z_positions = [
i * 1000000 + 1000 for i in range(self.simulate_data.n_duts)
] # 1m distance to see scattering better
self.simulate_data.dut_pixel_size = [
(1, 1)
] * self.simulate_data.n_duts # If the pixel size is too big this tests fails due to pixel discretisation error
self.simulate_data.dut_n_pixel = [(10000, 10000)
] * self.simulate_data.n_duts
self.simulate_data.shuffle_hits = False
self.simulate_data.digitization_charge_sharing = False # Simplify position reconstruction
def gauss(x, A, mu, sigma): # Scattering angle theta fit function
return A * np.exp(-(x - mu)**2 / (2. * sigma**2))
def check_scattering_angle():
# Expected scattering angle theta_0
theta_0 = self.simulate_data._scattering_angle_sigma(
self.simulate_data.dut_material_budget[0],
charge_number=1) * 1000
# Extract theta from simulation results by using the hit positions
with tb.open_file('simulated_data_DUT0.h5', 'r') as in_file_h5:
dut0_x = (in_file_h5.root.Hits[:]['column'] -
1) * self.simulate_data.dut_pixel_size[0][0]
dut0_y = (in_file_h5.root.Hits[:]['row'] -
1) * self.simulate_data.dut_pixel_size[0][1]
with tb.open_file('simulated_data_DUT1.h5', 'r') as in_file_h5:
dut1_x = (in_file_h5.root.Hits[:]['column'] -
1) * self.simulate_data.dut_pixel_size[1][0]
dut1_y = (in_file_h5.root.Hits[:]['row'] -
1) * self.simulate_data.dut_pixel_size[1][1]
# Calculate theta in spherical coordinates from data
dx = dut1_x.astype(np.float) - dut0_x.astype(np.int)
dy = dut1_y.astype(np.float) - dut0_y.astype(np.float)
dz = np.ones_like(dx) * (self.simulate_data.z_positions[1] -
self.simulate_data.z_positions[0])
_, theta, _ = geometry_utils.cartesian_to_spherical(dx, dy, dz)
if theta_0 == 0: # Special case: no material budget thus not scattering, theta has to be beam angle
self.assertTrue(
np.allclose(self.simulate_data.beam_angle,
theta * 1000.,
atol=0.01))
return
if self.simulate_data.beam_angle == 0:
hist, bins = np.histogram(
theta * 1000, range=(0, np.pi),
bins=1000) # Histogramm scattering angle distribution
x, y = (bins[:-1] + bins[1:]) / 2., hist
# import matplotlib.pyplot as plt
# plt.bar(x, y, width=np.diff(x)[0])
# plt.plot(x, gauss(x, *(np.amax(y), 0, theta_0)), 'r-', linewidth=2)
# plt.legend()
# plt.show()
# Fit scatterign distribution
coeff, _ = curve_fit(gauss,
x,
y,
p0=[np.amax(hist), 0,
theta_0]) # Fit theta distribution
self.assertTrue(
np.allclose(theta_0, coeff[2], atol=0.02),
'The scattering angle for multiple scattering is wrong'
) # Check for similarity, slight differences most likely due to pixel position binning
else: # TODO: this is maybe a bug, the theta distribution with starting beam angle does not look gaussian; just check the mean here;
self.assertTrue(
np.allclose(np.mean(theta * 1000.),
self.simulate_data.beam_angle,
atol=0.05),
'The beam direction with multiple scattering is wrong')
# Test 1: Check scattering for different device thickness from 50 um to 1000 um
for device_thickness in range(0, 1000, 50): # Change the thickness
self.simulate_data.dut_thickness = [device_thickness
] * self.simulate_data.n_duts
self.simulate_data.dut_material_budget = [
self.simulate_data.dut_thickness[i] * 1e-4 / 9.370
for i in range(self.simulate_data.n_duts)
] # Assume silicon sensor
self.simulate_data.create_data_and_store('simulated_data',
n_events=100000)
check_scattering_angle()
# Test 2: Check scattering for different device z positions
for z_position in ([[i, i + 1000000]
for i in range(0, 1000000, 250000)
]): # Put planes at different positions
self.simulate_data.z_positions = z_position
self.simulate_data.create_data_and_store('simulated_data',
n_events=100000)
check_scattering_angle()
# Test 3: Check with beam angle
for beam_angle in range(5):
self.simulate_data.beam_angle = beam_angle
self.simulate_data.z_positions = [0, 100000]
self.simulate_data.dut_thickness = [1000
] * self.simulate_data.n_duts
self.simulate_data.dut_material_budget = [
self.simulate_data.dut_thickness[i] * 1e-4 / 9.370
for i in range(self.simulate_data.n_duts)
]
self.simulate_data.create_data_and_store('simulated_data',
n_events=100000)
check_scattering_angle()
def test_dut_rotation(self):
self.simulate_data.reset()
# Set two planes and check the scattering angle due to the material budget of the first plane
self.simulate_data.n_duts = 2
self.simulate_data.set_std_settings()
self.simulate_data.tracks_per_event_sigma = 0
self.simulate_data.beam_angle = 0
self.simulate_data.beam_angle_sigma = 0
self.simulate_data.dut_pixel_size = [
(1, 1)
] * self.simulate_data.n_duts # If the pixel size is too big this tests fails due to pixel discretisation error
self.simulate_data.dut_n_pixel = [(100000, 100000)
] * self.simulate_data.n_duts
self.simulate_data.z_positions = [
i * 1000000. + 1000. for i in range(self.simulate_data.n_duts)
] # 1m distance to avoid intersecting DUT planes
self.simulate_data.tracks_per_event = 1
self.simulate_data.tracks_per_event_sigma = 0
self.simulate_data.digitization_charge_sharing = False # Not needed, save time
self.simulate_data.dut_material_budget = [
0
] * self.simulate_data.n_duts # Turn off multiple scattering
def check_rotations(
): # Rotations are checked by column / row correlation in the global coordinate system at 0,0,0 of the first DUT; correlation have to have the slope 1 and the offset 0
def line(x, c0, c1): # Correlation line fit
return c1 * x + c0
with tb.open_file('simulated_data_DUT0.h5', 'r') as in_file_1_h5:
with tb.open_file('simulated_data_DUT1.h5',
'r') as in_file_2_h5:
# Merge data on event basis for correlation, since one cannot assume that every plane is always hit
hits_0, hits_1 = analysis_utils.merge_on_event_number(
in_file_1_h5.root.Hits[:], in_file_2_h5.root.Hits[:])
# Get transformation matrix (translation + rotation) from simulation settting
transformation_matrix = geometry_utils.local_to_global_transformation_matrix(
x=self.simulate_data.offsets[1][0] -
self.simulate_data.offsets[0][0],
y=self.simulate_data.offsets[1][1] -
self.simulate_data.offsets[0][1],
z=self.simulate_data.z_positions[1] -
self.simulate_data.z_positions[0],
alpha=self.simulate_data.rotations[1][0],
beta=self.simulate_data.rotations[1][1],
gamma=self.simulate_data.rotations[1][2],
)
hits_1_column_global, hits_1_row_global, _ = geometry_utils.apply_transformation_matrix(
hits_1['column'], hits_1['row'],
np.zeros_like(hits_1['column']), transformation_matrix)
coeff_column, _ = curve_fit(line,
hits_1_column_global,
hits_0['column'],
p0=[0, 1.])
coeff_row, _ = curve_fit(line,
hits_1_row_global,
hits_0['row'],
p0=[0, 1.])
# Check column / row relative offsets are from line fit offsets with 1 mu precision
if hits_1_column_global.shape[
0] > 25: # TODO: Might be a bug that sometimes not many hits are created or just geometry
self.assertAlmostEqual(coeff_column[0], 0., delta=1)
self.assertAlmostEqual(coeff_row[0], 0., delta=1)
# Check alpha / beta angles with from line fit slopes with 0.1 % precision
self.assertAlmostEqual(coeff_column[1],
1.,
delta=0.001)
self.assertAlmostEqual(coeff_row[1], 1., delta=0.001)
# plt.plot(hits_1_column_global, hits_0['column'], 'o', label='Data')
# plt.plot(hits_1_column_global, line(hits_1_column_global, *coeff_column), '--', label='Fit')
# plt.plot(hits_1_row_global, hits_0['row'], 'o', label='Data row')
# plt.plot(hits_1_row_global, line(hits_1_row_global, *coeff_row), '--', label='Fit row')
# plt.legend(loc=0)
# plt.show()
# Test: Check correlation for different alpha, beta, gamma angles (x/y/z-axis rotation) and different relative offsets in x/y between the planes
for alpha in [0, np.pi / 8., np.pi / 6., np.pi / 4., np.pi / 3.]:
for beta in [0, np.pi / 8., np.pi / 6., np.pi / 4., np.pi / 3.]:
for gamma in [
0, np.pi / 8., np.pi / 6., np.pi / 4., np.pi / 3.
]:
for offset_x in range(-1000, 1001, 500):
for offset_y in range(-1000, 1001, 500):
self.simulate_data.rotations[1] = (alpha, beta,
gamma)
self.simulate_data.offsets[1] = (
self.simulate_data.offsets[0][0] + offset_x,
self.simulate_data.offsets[0][1] + offset_y
) # Set x/y shift with respect to DUT 0
self.simulate_data.create_data_and_store(
'simulated_data', n_events=1000)
check_rotations()
def test_simulation(self):
''' Check the full simulation '''
self.simulate_data.reset()
self.simulate_data.set_std_settings()
self.assertEqual(self.simulate_data.n_duts, 6)
self.simulate_data.create_data_and_store('simulated_data',
n_events=10000)
for dut_index in range(self.simulate_data.n_duts):
data_equal, error_msg = test_tools.compare_h5_files(
'simulated_data_DUT%d.h5' % dut_index,
os.path.join(tests_data_folder,
'simulated_data_DUT%d.h5' % dut_index),
exact=False)
self.assertTrue(data_equal, msg=error_msg)
def test_start_fails_with_unknown_kwargs(self):
xvfb = Xvfb(foo='bar')
with self.assertRaises(RuntimeError):
xvfb.start()
class WPTAgent(object):
"""Main agent workflow"""
def __init__(self, options, browsers):
from internal.browsers import Browsers
from internal.webpagetest import WebPageTest
from internal.traffic_shaping import TrafficShaper
from internal.adb import Adb
from internal.ios_device import iOSDevice
self.must_exit = False
self.options = options
self.capture_display = None
self.job = None
self.task = None
self.xvfb = None
self.root_path = os.path.abspath(os.path.dirname(__file__))
self.wpt = WebPageTest(options, os.path.join(self.root_path, "work"))
self.persistent_work_dir = self.wpt.get_persistent_dir()
self.adb = Adb(self.options, self.persistent_work_dir) if self.options.android else None
self.ios = iOSDevice(self.options.device) if self.options.iOS else None
self.browsers = Browsers(options, browsers, self.adb, self.ios)
self.shaper = TrafficShaper(options)
atexit.register(self.cleanup)
signal.signal(signal.SIGTERM, self.signal_handler)
signal.signal(signal.SIGINT, self.signal_handler)
self.image_magick = {'convert': 'convert', 'compare': 'compare', 'mogrify': 'mogrify'}
if platform.system() == "Windows":
paths = [os.getenv('ProgramFiles'), os.getenv('ProgramFiles(x86)')]
for path in paths:
if path is not None and os.path.isdir(path):
dirs = sorted(os.listdir(path), reverse=True)
for subdir in dirs:
if subdir.lower().startswith('imagemagick'):
convert = os.path.join(path, subdir, 'convert.exe')
compare = os.path.join(path, subdir, 'compare.exe')
mogrify = os.path.join(path, subdir, 'mogrify.exe')
if os.path.isfile(convert) and \
os.path.isfile(compare) and \
os.path.isfile(mogrify):
if convert.find(' ') >= 0:
convert = '"{0}"'.format(convert)
if compare.find(' ') >= 0:
compare = '"{0}"'.format(compare)
if mogrify.find(' ') >= 0:
mogrify = '"{0}"'.format(mogrify)
self.image_magick['convert'] = convert
self.image_magick['compare'] = compare
self.image_magick['mogrify'] = mogrify
break
def run_testing(self):
"""Main testing flow"""
if (sys.version_info >= (3, 0)):
from time import monotonic
else:
from monotonic import monotonic
start_time = monotonic()
browser = None
exit_file = os.path.join(self.root_path, 'exit')
message_server = None
if not self.options.android and not self.options.iOS:
from internal.message_server import MessageServer
message_server = MessageServer()
message_server.start()
if not message_server.is_ok():
logging.error("Unable to start the local message server")
return
while not self.must_exit:
try:
self.alive()
if os.path.isfile(exit_file):
try:
os.remove(exit_file)
except Exception:
pass
self.must_exit = True
break
if message_server is not None and self.options.exit > 0 and \
not message_server.is_ok():
logging.error("Message server not responding, exiting")
break
if self.browsers.is_ready():
self.job = self.wpt.get_test(self.browsers.browsers)
if self.job is not None:
self.job['image_magick'] = self.image_magick
self.job['message_server'] = message_server
self.job['capture_display'] = self.capture_display
self.job['shaper'] = self.shaper
self.task = self.wpt.get_task(self.job)
while self.task is not None:
start = monotonic()
try:
self.task['running_lighthouse'] = False
if self.job['type'] != 'lighthouse':
self.run_single_test()
self.wpt.get_bodies(self.task)
if self.task['run'] == 1 and not self.task['cached'] and \
self.job['warmup'] <= 0 and \
self.task['error'] is None and \
'lighthouse' in self.job and self.job['lighthouse']:
if 'page_result' not in self.task or \
self.task['page_result'] is None or \
self.task['page_result'] == 0 or \
self.task['page_result'] == 99999:
self.task['running_lighthouse'] = True
self.wpt.running_another_test(self.task)
self.run_single_test()
elapsed = monotonic() - start
logging.debug('Test run time: %0.3f sec', elapsed)
except Exception as err:
msg = ''
if err is not None and err.__str__() is not None:
msg = err.__str__()
self.task['error'] = 'Unhandled exception running test: '\
'{0}'.format(msg)
logging.exception("Unhandled exception running test: %s", msg)
traceback.print_exc(file=sys.stdout)
self.wpt.upload_task_result(self.task)
# Set up for the next run
self.task = self.wpt.get_task(self.job)
if self.job is not None:
self.job = None
else:
self.sleep(self.options.polling)
except Exception as err:
msg = ''
if err is not None and err.__str__() is not None:
msg = err.__str__()
if self.task is not None:
self.task['error'] = 'Unhandled exception preparing test: '\
'{0}'.format(msg)
logging.exception("Unhandled exception: %s", msg)
traceback.print_exc(file=sys.stdout)
if browser is not None:
browser.on_stop_capture(None)
browser.on_stop_recording(None)
browser = None
if self.options.exit > 0:
run_time = (monotonic() - start_time) / 60.0
if run_time > self.options.exit:
break
# Exit if adb is having issues (will cause a reboot after several tries)
if self.adb is not None and self.adb.needs_exit:
break
self.cleanup()
def run_single_test(self):
"""Run a single test run"""
self.alive()
browser = self.browsers.get_browser(self.job['browser'], self.job)
if browser is not None:
browser.prepare(self.job, self.task)
browser.launch(self.job, self.task)
try:
if self.task['running_lighthouse']:
self.task['lighthouse_log'] = \
'Lighthouse testing is not supported with this browser.'
try:
browser.run_lighthouse_test(self.task)
except Exception:
logging.exception('Error running lighthouse test')
if self.task['lighthouse_log']:
try:
log_file = os.path.join(self.task['dir'], 'lighthouse.log.gz')
with gzip.open(log_file, GZIP_TEXT, 7) as f_out:
f_out.write(self.task['lighthouse_log'])
except Exception:
logging.exception('Error compressing lighthouse log')
else:
browser.run_task(self.task)
except Exception as err:
msg = ''
if err is not None and err.__str__() is not None:
msg = err.__str__()
self.task['error'] = 'Unhandled exception in test run: '\
'{0}'.format(msg)
logging.exception("Unhandled exception in test run: %s", msg)
traceback.print_exc(file=sys.stdout)
browser.stop(self.job, self.task)
# Delete the browser profile if needed
if self.task['cached'] or self.job['fvonly']:
browser.clear_profile(self.task)
else:
err = "Invalid browser - {0}".format(self.job['browser'])
logging.critical(err)
self.task['error'] = err
browser = None
def signal_handler(self, *_):
"""Ctrl+C handler"""
if self.must_exit:
exit(1)
if self.job is None:
print("Exiting...")
else:
print("Will exit after test completes. Hit Ctrl+C again to exit immediately")
self.must_exit = True
def cleanup(self):
"""Do any cleanup that needs to be run regardless of how we exit"""
logging.debug('Cleaning up')
self.shaper.remove()
if self.xvfb is not None:
self.xvfb.stop()
if self.adb is not None:
self.adb.stop()
if self.ios is not None:
self.ios.disconnect()
def sleep(self, seconds):
"""Sleep wrapped in an exception handler to properly deal with Ctrl+C"""
try:
time.sleep(seconds)
except IOError:
pass
def wait_for_idle(self, timeout=30):
"""Wait for the system to go idle for at least 2 seconds"""
if (sys.version_info >= (3, 0)):
from time import monotonic
else:
from monotonic import monotonic
import psutil
logging.debug("Waiting for Idle...")
cpu_count = psutil.cpu_count()
if cpu_count > 0:
target_pct = 50. / float(cpu_count)
idle_start = None
end_time = monotonic() + timeout
idle = False
while not idle and monotonic() < end_time:
self.alive()
check_start = monotonic()
pct = psutil.cpu_percent(interval=0.5)
if pct <= target_pct:
if idle_start is None:
idle_start = check_start
if monotonic() - idle_start > 2:
idle = True
else:
idle_start = None
def alive(self):
"""Touch a watchdog file indicating we are still alive"""
if self.options.alive:
with open(self.options.alive, 'a'):
os.utime(self.options.alive, None)
def requires(self, module, module_name=None):
"""Try importing a module and installing it if it isn't available"""
ret = False
if module_name is None:
module_name = module
try:
__import__(module)
ret = True
except ImportError:
pass
if not ret and sys.version_info < (3, 0):
from internal.os_util import run_elevated
logging.debug('Trying to install %s...', module_name)
subprocess.call([sys.executable, '-m', 'pip', 'uninstall', '-y', module_name])
run_elevated(sys.executable, '-m pip uninstall -y {0}'.format(module_name))
subprocess.call([sys.executable, '-m', 'pip', 'install', module_name])
run_elevated(sys.executable, '-m pip install {0}'.format(module_name))
try:
__import__(module)
ret = True
except ImportError:
pass
if not ret:
if (sys.version_info >= (3, 0)):
print("Missing {0} module. Please run 'pip3 install {1}'".format(module, module_name))
else:
print("Missing {0} module. Please run 'pip install {1}'".format(module, module_name))
return ret
def startup(self, detected_browsers):
"""Validate that all of the external dependencies are installed"""
ret = True
# default /tmp/wptagent as an alive file on Linux
if self.options.alive is None:
if platform.system() == "Linux":
self.options.alive = '/tmp/wptagent'
else:
self.options.alive = os.path.join(os.path.dirname(__file__), 'wptagent.alive')
self.alive()
ret = self.requires('dns', 'dnspython') and ret
ret = self.requires('monotonic') and ret
ret = self.requires('PIL', 'pillow') and ret
ret = self.requires('psutil') and ret
ret = self.requires('requests') and ret
if not self.options.android and not self.options.iOS:
ret = self.requires('tornado') and ret
# Windows-specific imports
if platform.system() == "Windows":
ret = self.requires('win32api', 'pywin32') and ret
if self.options.webdriver and 'Firefox' in detected_browsers:
ret = self.requires('selenium')
# Optional imports
self.requires('brotli')
self.requires('fontTools', 'fonttools')
# Try patching ws4py with a faster lib
try:
self.requires('wsaccel')
import wsaccel
wsaccel.patch_ws4py()
except Exception:
logging.debug('wsaccel not installed, Chrome debug interface will be slower than it could be')
try:
subprocess.check_output([sys.executable, '--version'])
except Exception:
print("Unable to start python.")
ret = False
try:
subprocess.check_output('{0} -version'.format(self.image_magick['convert']), shell=True)
except Exception:
print("Missing convert utility. Please install ImageMagick and make sure it is in the path.")
ret = False
try:
subprocess.check_output('{0} -version'.format(self.image_magick['mogrify']), shell=True)
except Exception:
print("Missing mogrify utility. Please install ImageMagick and make sure it is in the path.")
ret = False
if platform.system() == "Linux":
try:
subprocess.check_output(['traceroute', '--version'])
except Exception:
logging.debug("Traceroute is missing, installing...")
subprocess.call(['sudo', 'apt', '-yq', 'install', 'traceroute'])
if self.options.webdriver and 'Firefox' in detected_browsers:
try:
subprocess.check_output(['geckodriver', '-V'])
except Exception:
logging.debug("geckodriver is missing, installing...")
subprocess.call(['sudo', 'apt', '-yq', 'install', 'firefox-geckodriver'])
# If we are on Linux and there is no display, enable xvfb by default
if platform.system() == "Linux" and not self.options.android and \
not self.options.iOS and 'DISPLAY' not in os.environ:
self.options.xvfb = True
if self.options.xvfb:
ret = self.requires('xvfbwrapper') and ret
if ret:
from xvfbwrapper import Xvfb
self.xvfb = Xvfb(width=1920, height=1200, colordepth=24)
self.xvfb.start()
# Figure out which display to capture from
if platform.system() == "Linux" and 'DISPLAY' in os.environ:
logging.debug('Display: %s', os.environ['DISPLAY'])
self.capture_display = os.environ['DISPLAY']
elif platform.system() == "Darwin":
proc = subprocess.Popen('ffmpeg -f avfoundation -list_devices true -i ""',
stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
_, err = proc.communicate()
for line in err.splitlines():
matches = re.search(r'\[(\d+)\] Capture screen', line.decode('utf-8'))
if matches:
self.capture_display = matches.group(1)
break
elif platform.system() == "Windows":
self.capture_display = 'desktop'
# Fix Lighthouse install permissions
if platform.system() != "Windows" and sys.version_info < (3, 0):
from internal.os_util import run_elevated
run_elevated('chmod', '-R 777 ~/.config/configstore/')
try:
import getpass
run_elevated('chown', '-R {0}:{0} ~/.config'.format(getpass.getuser()))
except Exception:
pass
# Check for Node 10+
if self.get_node_version() < 10.0:
if platform.system() == "Linux":
# This only works on debian-based systems
logging.debug('Updating Node.js to 12.x')
subprocess.call('curl -sL https://deb.nodesource.com/setup_12.x | sudo -E bash -',
shell=True)
subprocess.call(['sudo', 'apt-get', 'install', '-y', 'nodejs'])
if self.get_node_version() < 10.0:
logging.warning("Node.js 10 or newer is required for Lighthouse testing")
# Check the iOS install
if self.ios is not None:
ret = self.ios.check_install()
if not self.options.android and not self.options.iOS and not self.options.noidle:
self.wait_for_idle(300)
if self.adb is not None:
if not self.adb.start():
print("Error configuring adb. Make sure it is installed and in the path.")
ret = False
self.shaper.remove()
if not self.shaper.install():
if platform.system() == "Windows":
print("Error configuring traffic shaping, make sure secure boot is disabled.")
else:
print("Error configuring traffic shaping, make sure it is installed.")
ret = False
# Update the Windows root certs
if platform.system() == "Windows":
self.update_windows_certificates()
return ret
def get_node_version(self):
"""Get the installed version of Node.js"""
version = 0
try:
if (sys.version_info >= (3, 0)):
stdout = subprocess.check_output(['node', '--version'], encoding='UTF-8')
else:
stdout = subprocess.check_output(['node', '--version'])
matches = re.match(r'^v(\d+\.\d+)', stdout)
if matches:
version = float(matches.group(1))
except Exception:
pass
return version
def update_windows_certificates(self):
""" Update the root Windows certificates"""
try:
cert_file = os.path.join(self.persistent_work_dir, 'root_certs.sst')
if not os.path.isdir(self.persistent_work_dir):
os.makedirs(self.persistent_work_dir)
needs_update = True
if os.path.isfile(cert_file):
days = (time.time() - os.path.getmtime(cert_file)) / 86400
if days < 5:
needs_update = False
if needs_update:
logging.debug("Updating Windows root certificates...")
if os.path.isfile(cert_file):
os.unlink(cert_file)
from internal.os_util import run_elevated
run_elevated('certutil.exe', '-generateSSTFromWU "{0}"'.format(cert_file))
if os.path.isfile(cert_file):
run_elevated('certutil.exe', '-addstore -f Root "{0}"'.format(cert_file))
except Exception:
pass
def get_all_user_tweets(screen_name, start, end, tweet_lim=3200, no_rt=True,
virtuald=False):
"""
Params
------
screen_name : str
start : datetime-obj
end : datetime-obj
no_rt : bool
tweet_lim : int {default 3,200}
returns
-------
{int} total number of tweet ids obtained
"""
# Make dir structure
makedir(screen_name)
# name of file for saving tweet ids
fname_tweet_ids = 'users/{0}/usr_tweetids_{0}.jsonl'.format(screen_name)
# Headless displays with Xvfb (X virtual framebuffer)
if virtuald:
vdisplay = Xvfb()
vdisplay.start()
# Selenium parames
delay = 1 # time to wait on each page load before reading the page
driver = webdriver.Chrome()
ids_total = 0
for day in range((end - start).days + 1):
# Get Twitter search url
startDate = increment_day(start, 0)
endDate = increment_day(start, 1)
url = twitter_url(screen_name, no_rt, startDate, endDate)
driver.get(url)
time.sleep(delay)
try:
found_tweets = \
driver.find_elements_by_css_selector('li.js-stream-item')
increment = 10
# Scroll through the Twitter search page
while len(found_tweets) >= increment:
# scroll down for more results
driver.execute_script(
'window.scrollTo(0, document.body.scrollHeight);'
)
time.sleep(delay)
# select tweets
found_tweets = driver.find_elements_by_css_selector(
'li.js-stream-item'
)
increment += 10
# Get the IDs for all Tweets
ids = []
with open(fname_tweet_ids, 'a') as fout:
for tweet in found_tweets:
try:
# get tweet id
tweet_id = tweet.find_element_by_css_selector(
'.time a.tweet-timestamp'
).get_attribute('href').split('/')[-1]
ids.append(tweet_id)
ids_total += 1
# break if tweet_lim has been reached
if ids_total == tweet_lim:
print('{} tweets found.'.format(ids_total))
return ids_total
except StaleElementReferenceException as e:
print('Lost element reference.', tweet)
# Save ids to file
data_to_write = list(set(ids))
fout.write(json.dumps(data_to_write)+'\n')
except NoSuchElementException:
print('No tweets found.')
start = increment_day(start, 1)
# Close selenium driver
driver.close()
if virtuald:
vdisplay.stop()
print('{} tweets found.'.format(ids_total))
return ids_total
def start_xvfb(): from xvfbwrapper import Xvfb global _xvfb _xvfb = Xvfb() _xvfb.start() atexit.register(_xvfb.stop)
continue
else:
break
return
def check_exists_by_xpath(xpath):
try:
browser.find_element_by_xpath(xpath)
except NoSuchElementException:
return False
return True
display = Xvfb()
display.start()
browser = webdriver.Chrome("bin/chromedriver", options=opts)
# Go to the Google home page
load_page(browser, URL)
time.sleep(15)
# Access requests via the `requests` attribute
URL_STREAM = ""
for request in browser.requests:
if request.response and "m3u8" in request.path:
URL_STREAM = request.path
with open("url_stream.txt", "w") as f:
f.write(URL_STREAM)
browser.close()
# 改变配置文件的复制版本.
desired_capabilities['proxy'] = {
"httpProxy": PROXY,
"ftpProxy": PROXY,
"sslProxy": PROXY,
"noProxy": None,
"proxyType": "MANUAL",
"class": "org.openqa.selenium.Proxy",
"autodetect": False
}
#使用新实例驱动
options = webdriver.ChromeOptions()
options.add_argument('--headless')
#无窗口启动Chrome
xvfb = Xvfb()
xvfb.start()
driver = webdriver.Chrome(chrome_options=options,
desired_capabilities=desired_capabilities)
#尝试访问登陆页面，登录页面有你的代理IP
driver.get(page_url)
#打印访问网页的title和源代码
print driver.title
print driver.page_source
#关闭
driver.quit()
xvfb.stop()
import time
# start an Xvfb virtual display if we can (if xvfbwrapper is installed)
try:
from xvfbwrapper import Xvfb
except (ModuleNotFoundError, ImportError):
print("Using physical display")
vdisplay = None
else:
print("Using Xvfb virtual display")
vdisplay = Xvfb(width=1024, height=768, colordepth=24)
vdisplay.start()
# wait a bit to make sure the virtual display is ready
time.sleep(1)
try:
import pathlib
import tempfile
import threading
import gi
gi.require_version("Gtk", "3.0")
gi.require_version("PangoCairo", "1.0")
from gi.repository import GLib
from gnuradio import gr
from gnuradio.grc.gui.Platform import Platform
from gnuradio.grc.gui.Application import Actions, Application
# include the necessary boilerplate from grc's main function to create the app
platform = Platform(
def get_girl(msg):
display = Xvfb(width=1200, height=600, colordepth=16)
display.start()
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('window-size=1200x600')
chrome_options.add_argument('--no-sandbox')
driver = webdriver.Chrome('/usr/local/bin/chromedriver',
chrome_options=chrome_options)
print('Getting website')
driver.get('http://make.girls.moe/')
driver.get_screenshot_as_file('../www/debug.png')
print('Got')
xpath_dict = {
'hair_color':
'//*[@id="root"]/div/div/div/div/div[1]/div[2]/div[2]/div/div[2]/div[1]/div',
'hair_style':
'//*[@id="root"]/div/div/div/div/div[1]/div[2]/div[2]/div/div[2]/div[2]/div',
'eye_color':
'//*[@id="root"]/div/div/div/div/div[1]/div[2]/div[2]/div/div[2]/div[3]/div',
'blush':
'//*[@id="root"]/div/div/div/div/div[1]/div[2]/div[2]/div/div[3]/div[1]/div',
'smile':
'//*[@id="root"]/div/div/div/div/div[1]/div[2]/div[2]/div/div[3]/div[2]/div',
'open_mouth':
'//*[@id="root"]/div/div/div/div/div[1]/div[2]/div[2]/div/div[3]/div[3]/div',
'hat':
'//*[@id="root"]/div/div/div/div/div[1]/div[2]/div[2]/div/div[3]/div[4]/div',
'ribbon':
'//*[@id="root"]/div/div/div/div/div[1]/div[2]/div[2]/div/div[3]/div[5]/div',
'glasses':
'//*[@id="root"]/div/div/div/div/div[1]/div[2]/div[2]/div/div[3]/div[6]/div',
'generate_btn':
'//*[@id="root"]/div/div/div/div/div[1]/div[2]/div[1]/div/button',
'image':
'//*[@id="root"]/div/div/div/div/div[1]/div[2]/div[1]/div/div/div/div/img',
'progress': '//*[@id="root"]/div/div/div/div/div[1]/div[1]'
}
user_options = parse_message_to_options(msg)
print(user_options)
# Check connectivity
progress = driver.find_element_by_xpath(xpath_dict['progress'])
print(progress)
print(progress.text)
if 'Network Error' in progress.text:
driver.get_screenshot_as_file('../www/debug.png')
driver.close()
display.stop()
return 'Network error'
# Select options
for option, choice in user_options.items():
try:
xpath = xpath_dict[option]
element = driver.find_element_by_xpath(xpath)
element.click()
try:
subelement = element.find_elements_by_link_text(choice)[0]
except:
subelements = element.find_elements_by_class_name('btn')
choices = element.text.split('\n')
subelement = subelements[choices.index(choice)]
subelement.click()
print('Element clicked')
print('Saving screenshot')
driver.get_screenshot_as_file('../www/debug.png')
except Exception as e:
print(e)
print('Unable to select {} for {}'.format(choice, option))
image_src = None
clicked = False
for i in range(20):
print('Waiting for 1 second')
time.sleep(5)
try:
print('Trying to download the image')
print('Trying to click the generate button')
generate_btn = driver.find_element_by_xpath(
xpath_dict['generate_btn'])
print(generate_btn)
if generate_btn.is_enabled() and not clicked:
generate_btn.click()
clicked = True
print('Button clicked')
if clicked:
print('Button clicked')
image = driver.find_element_by_xpath(xpath_dict['image'])
image_src = image.get_attribute('src')
print('Successful')
break
except Exception as e:
print('Failed for the {} th time'.format(i))
print(e)
finally:
driver.get_screenshot_as_file('../www/debug.png')
with open('../www/animation.html', 'w') as f:
f.write('<img src="{}"/>'.format(image_src))
print('Writing result image to animation.html')
driver.close()
display.stop()
return 'http://www.bowendeng.com/animation.html' if image_src else 'failed'
