def jupyter_play(env, agent, steps=1000): from IPython import display policy = agent.get_policy() vdisplay = Xvfb(width=1280, height=740) vdisplay.start() observation = env.reset() def showarray(a, fmt='png'): a = np.uint8(a) f = io.BytesIO() ima = PIL.Image.fromarray(a).save(f, fmt) return f.getvalue() imagehandle = display.display(display.Image(data=showarray(env.render(mode='rgb_array')), width=450), display_id='gymscr') action = 0 reward = 0 for _ in range(steps): time.sleep(0.001) action = agent.compute_action(observation, prev_reward=reward, prev_action=action) observation, reward, done, info = env.step(action) display.update_display(display.Image(data=showarray(env.render(mode='rgb_array')), width=450), display_id='gymscr') if done: break vdisplay.stop()
def get_seller_phone(url): # телефон показывается в виде картинки, используем selenium и pytesseract vdisplay = Xvfb() vdisplay.start() driver = webdriver.Chrome(options=options) driver.set_window_size(1920, 1080) driver.get(url) try: button = driver.find_element_by_xpath('//a[@class="button item-phone-button js-item-phone-button ' 'button-origin button-origin-blue button-origin_full-width ' 'button-origin_large-extra item-phone-button_hide-phone ' 'item-phone-button_card js-item-phone-button_card"]') button.click() time.sleep(2) driver.save_screenshot("phone_number.png") image = driver.find_element_by_xpath('//div[@class="item-phone-big-number js-item-phone-big-number"]//*') cropped = Image.open("phone_number.png") x, y = image.location["x"], image.location["y"] width, height = image.size["width"], image.size["height"] cropped.crop((x, y, x + width, y + height)).save("phone.gif") phone = Image.open("phone.gif") phone_text = image_to_string(phone) except Exception as e: with open("logs.txt", "a", encoding="utf8") as file: file.write(str(e) + " avito get_seller_phone\n") phone_text = "Не указано" driver.quit() vdisplay.stop() return phone_text
def runner(context): driverManager = context['driverManager'] user = context['user'] console: Console = context['console'] playlist = context['playlist'] queueUrl = context['queueUrl'] proxy = context['proxy'] shutdownEvent = context['shutdownEvent'] pid = current_process().pid try: vdisplay = Xvfb(width=1280, height=1024, colordepth=24, tempdir=None, noreset='+render') vdisplay.start() driverData = driverManager.getDriver(type='chrome', uid=pid, user=user, proxy=proxy) if not driverData: return driver = driverData['driver'] userDataDir = driverData['userDataDir'] if not driver: return except: console.error('Unavailale webdriver: %s' % format_exc()) else: try: spotify = Spotify.Adapter(driver, console, shutdownEvent) if not shutdownEvent.is_set(): if spotify.register(user): message = {'user': user, 'playlist': playlist} client = client('sqs') client.send_message( QueueUrl=queueUrl, MessageBody=dumps(message), DelaySeconds=1, ) except: console.exception() if driver: try: driver.quit() except: pass if userDataDir: try: rmtree(path=userDataDir, ignore_errors=True) except: pass if vdisplay: try: vdisplay.stop() except: pass
def handle(self, *args, **options): xvfb = Xvfb(width=1600, height=720) xvfb.start() browser = WebDriver() browser.get(settings.DOMAIN) sleep(1) browser.find_element_by_css_selector("#disclaimer button").click() sleep(0.7) browser.find_elements_by_css_selector(".officer .checkmark")[4].click() sleep(2) browser.find_element_by_css_selector( ".complaint-row .col-md-3").click() sleep(1) content = browser.get_screenshot_as_png() now = datetime.datetime.now().strftime("%Y-%m-%d %H-%M-%S") file_name = "{now}.png".format(now=now) file_path = os.path.join(settings.BASE_DIR, 'static', file_name) with open(file_path, "wb") as f: f.write(content) browser.quit() email = EmailMessage(subject='CPDB Homepage screenshot %s' % now, body='FYI', to=['*****@*****.**']) email.attach_file(file_path) email.send() xvfb.stop()
class Test(unittest.TestCase): def setUp(self): self.xvfb = os.environ.get("ENABLE_XVFB", False) self.browser = os.environ.get("BROWSER", "Chrome") if self.xvfb: self.vdisplay = Xvfb(width=1280, height=720) self.vdisplay.start() if self.browser == "Firefox": self.driver = self.get_ff_driver() else: self.driver = self.get_chrome_driver() self.load = self.driver.get def tearDown(self): if self.driver: self.driver.quit() if self.xvfb and self.vdisplay: self.vdisplay.stop() def get_ff_driver(self): return webdriver.Firefox() def get_chrome_driver(self): opts = Options() if "TRAVIS" in os.environ: # github.com/travis-ci/travis-ci/issues/938 opts.add_argument("--no-sandbox") # Fix for https://code.google.com/p/chromedriver/issues/detail?id=799 opts.add_experimental_option("excludeSwitches", ["ignore-certificate-errors"]) return webdriver.Chrome(chrome_options=opts) def test_should_load_wikipedia(self): self.load("https://www.wikipedia.org") self.assertIn("Wikipedia", self.driver.title)
def get_apartment_data(url): vdisplay = Xvfb() vdisplay.start() driver = webdriver.Chrome(options=options) driver.set_window_size(1920, 1080) driver.get(url) city, district, street, block_number = get_address(driver) sell_type, rent_info = get_selling_type(url) if "продажа" in sell_type.lower(): rent_info = "Не аренда" material, lift, year, rooms_number, floor, total_floors, total_area, kitchen_area, repair = get_apartment_params(driver) block_type = "Вторичка" living_area = "Не указано" price = get_price(driver) if "Аренда" in sell_type: if "posutochno" in url: price += "/день" else: price += "/мес." #seller_type, seller_name = get_seller_info(driver) images = get_photos(driver) description = get_description(driver) phone = get_seller_phone(driver) selling_detail = "Не указано" driver.quit() vdisplay.stop() return [city, district, street, block_number, sell_type, rent_info, price, block_type, rooms_number, total_area, total_floors, material, selling_detail, images, description, phone, kitchen_area, living_area, floor]
def main(): import sys import imageio import pandas as pd import matplotlib.pyplot as plt from xvfbwrapper import Xvfb plt.ioff() # with Xvfb() as xvfb: # plt.ioff() vdisplay = Xvfb() vdisplay.start() if len(sys.argv)<5: subsampleRate = 10 else: subsampleRate = int(sys.argv[4]) if len(sys.argv)<6: speedup = 10 else: speedup = int(sys.argv[5]) bounds2video(sys.argv[1],sys.argv[2],sys.argv[3],subsampleRate,speedup) vdisplay.stop()
def main(): import sys import imageio import pandas as pd import matplotlib.pyplot as plt from xvfbwrapper import Xvfb plt.ioff() # with Xvfb() as xvfb: # plt.ioff() vdisplay = Xvfb() vdisplay.start() if len(sys.argv) < 5: subsampleRate = 10 else: subsampleRate = int(sys.argv[4]) if len(sys.argv) < 6: speedup = 10 else: speedup = int(sys.argv[5]) bounds2video(sys.argv[1], sys.argv[2], sys.argv[3], subsampleRate, speedup) vdisplay.stop()
def tree_builder(node_file, function_folder): """ Perform tree builder script to extract csv, tsv and svg. To correctly run tree_builder needs Xorg. We are using xvfb-run to workaround on "x-server-less" systems. Script usage: python new_tree_builder_for_perl_tango.py -d visualization_ITSoneDB_r131.dmp """ sys.stdout.write('\n### Step 5: Tree Builder\n') # xvbf-run issue: # /bin/xvfb-run: line 181: 17066 Segmentation fault DISPLAY=:$SERVERNUM XAUTHORITY=$AUTHFILE "$@" 2>&1 # solution: https://unix.stackexchange.com/questions/152957/xvfb-run-aborting-on-cluster ###command = 'xvfb-run -d -e /tmp/biomas_xvfb.log '+__python_executable__+' '+__location__+'/new_tree_builder_for_perl_tango.py -d '+ visualization ###command = 'xvfb-run '+__python_executable__+' '+__location__+'/new_tree_builder_for_perl_tango.py -d '+ visualization #stdout, stderr, status = run_command(command) command = __python_executable__+' '+__location__+'/new_tree_builder_for_perl_tango.py -d '+ node_file + ' -F ' + function_folder sys.stdout.write(command) from xvfbwrapper import Xvfb vdisplay = Xvfb() vdisplay.start() stdout, stderr, status = run_command(command) vdisplay.stop() # Read stdout for errors if status == 0: sys.stdout.write(stdout) else: sys.stderr.write(stderr)
class BaseSlackParser: """Class for basic slack parser functionality. """ def __init__(self, base_addr, browser="firefox", executable_path="./bot/core/drivers/geckodriver", browser_window_size=(1920, 1080), page_load_timeout=30, sticky_timeout=30, headless=True): self._base_addr = base_addr self._headless = headless if self._headless: self.xvfb = Xvfb() self.xvfb.start() self.browser = Browser(browser, headless=False, wait_time=30, executable_path=executable_path) self.browser.driver.implicitly_wait(sticky_timeout) self.browser.driver.set_page_load_timeout(page_load_timeout) self.browser.driver.set_window_size(*browser_window_size) self.browser.visit(self._base_addr) def __del__(self): if self._headless: self.xvfb.stop() self.browser.quit()
class BaseTestCase(unittest.TestCase): def setUp(self): self.xvfb = os.environ.get("ENABLE_XVFB", False) self.browser = os.environ.get("BROWSER", "Chrome") if self.xvfb: self.vdisplay = Xvfb(width=1280, height=720) self.vdisplay.start() if self.browser == "Firefox": self.driver = self.get_ff_driver() else: self.driver = self.get_chrome_driver() self.load = self.driver.get def tearDown(self): if self.driver: self.driver.quit() if self.xvfb and self.vdisplay: self.vdisplay.stop() def get_ff_driver(self): return webdriver.Firefox() def get_chrome_driver(self): opts = Options() if "TRAVIS" in os.environ: # github.com/travis-ci/travis-ci/issues/938 opts.add_argument("--no-sandbox") # Fix for https://code.google.com/p/chromedriver/issues/detail?id=799 opts.add_experimental_option("excludeSwitches", ["ignore-certificate-errors"]) return webdriver.Chrome(chrome_options=opts)
class TestJavascript(unittest.TestCase): def setUp(self): # Configure a web service. handler = http.server.SimpleHTTPRequestHandler address = ("127.0.0.1", 9999) socketserver.TCPServer.allow_reuse_address = True # Prevents address conflicts. httpd = socketserver.TCPServer(address, handler) # Start the web service in a separate thread as deamon. httpd_thread = threading.Thread(target=httpd.serve_forever) httpd_thread.setDaemon(True) httpd_thread.start() # Start a display. self.display = Xvfb() self.display.start() # Start the browser driver for selenium testing. self.driver = webdriver.Firefox() self.driver.get("http://localhost:9999/index.html") def test_javascript(self): # Create an instance of the selenium Firefox driver. error = self.driver.find_elements_by_id("error")[0].text self.assertEqual(error, "") def tearDown(self): self.display.stop()
def run(): #driver = webdriver.Chrome(chrome_options=options) print('Sreencast website animation') xvfb = Xvfb(width=1280, height=720, colordepth=24) xvfb.start() options = webdriver.chrome.options.Options() options.add_argument("--no-sandbox") options.add_argument("--disable-setuid-sandbox") options.add_experimental_option("excludeSwitches", ["enable-automation"]) options.add_experimental_option('useAutomationExtension', False) options.add_argument("--kiosk") options.add_argument("--disable-extensions") browser = webdriver.Chrome(chrome_options=options) #browser = webdriver.Firefox() url = 'https://clickcall.co.nz/af939c60-79bd-4d92-9785-e58e3944bd4f' #destination = 'movie.flv' browser.get(url) # normal quality, lagging in the first part on the video. filesize ~7MB # ffmpeg_stream = 'ffmpeg -f x11grab -s 1280x720 -r 24 -i :%d+nomouse -c:v libx264 -preset superfast -pix_fmt yuv420p -s 1280x720 -threads 0 -f flv "%s"' % (xvfb.new_display, destination) # high quality, no lagging but huge file size ~50MB ffmpeg_stream = 'ffmpeg -y -r 30 -f x11grab -s 1280x720 -probesize 42M -i :%d+nomouse -c:v libx264rgb -crf 15 -preset:v ultrafast -c:a pcm_s16le -af aresample=async=1:first_pts=0 /output/out.mkv' % xvfb.new_display #ffmpeg_stream = 'ffmpeg -f x11grab -s 1920x1080 -r 30 -i :0.0 -q:v 0 -c:v libx264rgb -crf fast -y /output/screen.mp4' % xvfb.new_display args = shlex.split(ffmpeg_stream) p = subprocess.Popen(args) print("-------------------------------") print(p) print("-------------------------------") time.sleep(30) # record for 30 secs browser.quit() xvfb.stop()
class OFMTransferSiteManager(OFMSiteManager): def __init__(self, user=None): # pylint: disable=super-init-not-called self.user = user if self.user: self._login_user = self.user.ofm_username self._login_password = self.user.ofm_password else: self._login_user = os.environ('OFM_USERNAME') self._login_password = os.environ('OFM_PASSWORD') self.display = Xvfb() self.display.start() def download_transfer_excels(self, matchdays=None): profile = webdriver.FirefoxProfile( os.path.join(BASE_DIR, 'ofm_transfer_data', 'firefox_profile')) profile.set_preference("browser.download.dir", os.path.join(BASE_DIR, 'ofm_transfer_data')) self.browser = webdriver.Firefox(firefox_profile=profile) self.browser.set_page_load_timeout(10) self.login() if not matchdays: matchdays = [Matchday.get_current()] for matchday in matchdays: if not self._is_transfer_file_present(matchday): try: self._jump_to_transfer_page(self, matchday=matchday) # pylint: disable=redundant-keyword-arg except TimeoutError: pass @staticmethod def _is_transfer_file_present(matchday=None): if not matchday: matchday = Matchday.get_current() if os.path.isfile( os.path.join( BASE_DIR, 'ofm_transfer_data', 'ofm_spielerwechsel_{}_{}.csv'.format( matchday.season.number, matchday.number))): return True return False @timeout(5, use_signals=False) def _jump_to_transfer_page(self, matchday=None): if not matchday: self.jump_to_frame(Constants.Transfer.DOWNLOAD_TRANSFERS) else: self.jump_to_frame( Constants.Transfer.DOWNLOAD_TRANSFERS_FROM_MATCHDAY.format( matchday.number)) def kill_browser(self): if self.browser: self.browser.stop_client() self.display.stop()
class Webdriver(unittest.TestCase): def setUp(self): self.vdisplay = Xvfb(width=1280, height=720) self.vdisplay.start() # if(not vdisplay.start()): # fo = open(LOG_FILE, "a") # fo.write("Xvfbfailure||"+str(TREATMENTID)+"||"+str(ID)+"\n") # fo.close() # sys.exit(0) if (BROWSER == 'firefox'): if (platform.system() == 'Darwin'): self.driver = webdriver.Firefox() elif (platform.system() == 'Linux'): self.driver = webdriver.Firefox(proxy=proxy) else: print "Unidentified Platform" sys.exit(0) elif (BROWSER == 'chrome'): print "WARNING: Expecting chromedriver at specified location !!" if (platform.system() == 'Darwin'): chromedriver = "./experiment/chromedriver/chromedriver_mac" os.environ["webdriver.chrome.driver"] = chromedriver self.driver = webdriver.Chrome(executable_path=chromedriver) elif (platform.system() == 'Linux'): chromedriver = "./experiment/chromedriver/chromedriver_linux" os.environ["webdriver.chrome.driver"] = chromedriver chrome_option = webdriver.ChromeOptions() chrome_option.add_argument( "--proxy-server=yogi.pdl.cmu.edu:3128") self.driver = webdriver.Chrome(executable_path=chromedriver, chrome_options=chrome_option) else: print "Unidentified Platform" sys.exit(0) else: print "Unsupported Browser" sys.exit(0) self.driver.implicitly_wait(10) self.base_url = "https://www.google.com/" self.verificationErrors = [] self.driver.set_page_load_timeout(40) self.accept_next_alert = True def test_webdriver(self): driver = self.driver cole.optIn(driver) driver.get(SITE) time.sleep(20) #raw_input("wait") pref = cole.get_ad_pref(driver) print SITE print 'pref=', pref if pref != []: fo = open(TARGET_FILE, "a") fo.write(SITE + "||" + "@".join(pref) + '\n') fo.close() def tearDown(self): self.vdisplay.stop() self.driver.quit()
def work(): while True: try: clone = requests.get("{}/api/clone".format(url)).json() print(clone) vdisplay = Xvfb() vdisplay.start() display = Display(visible=0, size=(800, 600)) display.start() driver = helper._init(clone['ip'], clone['port'], clone['c_user'], clone['xs']) try: helper.newest_message(driver) helper.request_message(driver) #active_user_links = helper.active_conversations(driver) #helper.message_to_active_users(driver, active_user_links) except Exception as e: print("Ex 1 : {}".format(e)) driver.save_screenshot('log-{}.{}'.format(clone['c_user'], 'png')) driver.quit() display.stop() vdisplay.stop() except Exception as e: print(e) time.sleep(5)
def plot_response(response_src, out_png=False): # start virtual display print("starting Xvfb"); vdisplay = Xvfb() vdisplay.start() response_src = np.loadtxt(src_txt) if len(response_src.shape) > 1: response_src = response_src[1] sphere = get_sphere('symmetric724') sh_resp = AxSymShResponse(0, response_src) sig_resp = sh_resp.on_sphere(sphere) sig_resp = sig_resp[None, None, None, :] ren = window.Renderer() sphere_actor = actor.odf_slicer(sig_resp, sphere=sphere,colormap='blues') ren.add(sphere_actor) my_camera = ren.camera() my_camera.SetPosition(1.62, -9.19, 4.01) my_camera.SetFocalPoint(0.01, -0.46, -0.19) my_camera.SetViewUp(0.24, 0.46, 0.86) if out_png != False: window.record(ren, out_path=out_png, magnification=10, size=(60, 60)) else: window.show(ren, reset_camera=False) print('Camera Settings') print('Position: ', '(%.2f, %.2f, %.2f)' % my_camera.GetPosition()) print('Focal Point: ', '(%.2f, %.2f, %.2f)' % my_camera.GetFocalPoint()) print('View Up: ', '(%.2f, %.2f, %.2f)' % my_camera.GetViewUp()) vdisplay.stop()
def enable(): import time import requests import settings from splinter import Browser from xvfbwrapper import Xvfb print "Trying to enable myself." vdisplay = Xvfb() vdisplay.start() email = settings.getEmail() password = settings.getPassword() team_name = settings.getTeamName() bot_user = settings.getBotUser() browser = Browser('chrome') url = 'https://{}.slack.com/services/{}'.format(team_name, bot_user) browser.visit(url) browser.fill('email', email) browser.fill('password', password) browser.find_by_id('signin_btn').first.click() browser.find_link_by_text('Enable').first.click() time.sleep(2) # Sometimes I saw a crash where there was no alert, so we'll wait a bit first. alert = browser.get_alert() alert.accept() time.sleep(2) # If you close the display too quickly, the request doesn't get processed. vdisplay.stop()
def worker(game, play_game, solution_q, reward_q, stop, show_screen): """ A worker that plays the game with a proposed solution. Workers fetch proposed solutions from solution_q (Solution Queue) and put the matching reward into the reward_q (Reward Queue). If a stop signal is received the worker terminates. Rewards are matched to solutions at the receiving and by using the solution_id as an identifier. """ if not show_screen: vdisplay = Xvfb() vdisplay.start() while not stop.is_set(): try: (solution_id, solution) = solution_q.get(False, 0.1) except queue.Empty: pass else: name = multiprocessing.current_process().name logger.info("%s started working on %s", name, solution_id) reward = play_game(solution=solution) reward_q.put((solution_id, reward)) if not show_screen: vdisplay.stop()
def tree_builder(visualization): """ Perform tree builder script to extract csv, tsv and svg. To correctly run tree_builder needs Xorg. We are using xvfb-run to workaround on "x-server-less" systems. Script usage: python new_tree_builder_for_perl_tango.py -d visualization_ITSoneDB_r131.dmp """ sys.stdout.write('\n### Step 5: Tree Builder\n') #command = 'xvfb-run -a -e /export/xvfb.log '+__python_executable__+' '+__location__+'/new_tree_builder_for_perl_tango.py -d '+ visualization #stdout, stderr, status = run_command(command) command = __python_executable__+' '+__location__+'/new_tree_builder_for_perl_tango.py -d '+ visualization from xvfbwrapper import Xvfb vdisplay = Xvfb() vdisplay.start() stdout, stderr, status = run_command(command) vdisplay.stop() # Read stdout for errors if status == 0: sys.stdout.write(stdout) else: sys.stderr.write(stdout) sys.stderr.write(stderr)
class Xvfb(Plugin): def options(self, parser, env): super(Xvfb, self).options(parser, env) parser.add_option("--with-xvfb-options", action="store", dest="xvfb_options", default=env.get("NOSE_WITH_XVFB_OPTIONS"), help="Options to pass to Xvfb. Comma delimited with " "equals as separators if necessary. " "E.g. \"extension=SELINUX, once\". Currently, there is no " "way to provide options that begin with a +. This is a limitation " "in xvfbwrapper. Repetition is not allowed. [NOSE_WITH_XVFB_OPTIONS]") def configure(self, options, noseconfig): super(Xvfb, self).configure(options, noseconfig) self.xvfb_options = {} if options.xvfb_options: opts = [x.strip() for x in options.xvfb_options.split(",")] for item in opts: key, sign, value = item.partition("=") if not value: value = '' self.xvfb_options[key] = value def begin(self): logger.info('Starting xvfb virtual display 1024x768 with %s' % self.xvfb_options) self.vdisplay = XvfbWrapper(width=1024, height=768, **self.xvfb_options) self.vdisplay.start() def finalize(self, result): logger.info('Stopping xvfb virtual display') self.vdisplay.stop()
def get_page(url): path_to_chromedriver = '/usr/bin/chromedriver' # change path as needed chrome_options = webdriver.ChromeOptions() chrome_options.add_argument('--no-sandbox') vdisplay = Xvfb() vdisplay.start() display = Display(visible=0, size=(1024, 768)) display.start() browser = webdriver.Chrome( executable_path=path_to_chromedriver, service_args=["--verbose", "--log-path=/tmp/CHROMIUM_LOG"], options=chrome_options) browser.get(url) time.sleep(2) page = browser.page_source browser.close() display.stop() vdisplay.stop() return page
class Webdriver(unittest.TestCase): def setUp(self): self.vdisplay = Xvfb(width=1280, height=720) self.vdisplay.start() # if(not vdisplay.start()): # fo = open(LOG_FILE, "a") # fo.write("Xvfbfailure||"+str(TREATMENTID)+"||"+str(ID)+"\n") # fo.close() # sys.exit(0) if(BROWSER=='firefox'): if (platform.system()=='Darwin'): self.driver = webdriver.Firefox() elif (platform.system()=='Linux'): self.driver = webdriver.Firefox(proxy=proxy) else: print "Unidentified Platform" sys.exit(0) elif(BROWSER=='chrome'): if (platform.system()=='Darwin'): chromedriver = "./experiment/chromedriver/chromedriver_mac" os.environ["webdriver.chrome.driver"] = chromedriver self.driver = webdriver.Chrome(executable_path=chromedriver) elif (platform.system() == 'Linux'): chromedriver = "./experiment/chromedriver/chromedriver_linux" os.environ["webdriver.chrome.driver"] = chromedriver chrome_option = webdriver.ChromeOptions() chrome_option.add_argument("--proxy-server=yogi.pdl.cmu.edu:3128" ) self.driver = webdriver.Chrome(executable_path=chromedriver, chrome_options=chrome_option) else: print "Unidentified Platform" sys.exit(0) else: print "Unsupported Browser" sys.exit(0) self.driver.implicitly_wait(10) self.base_url = "https://www.google.com/" self.verificationErrors = [] self.driver.set_page_load_timeout(40) self.accept_next_alert = True def test_webdriver(self): fo = open(AD_FILE, "w") fo.close() driver = self.driver driver.get(SITE) count = 0 while(count < N): els = driver.find_elements_by_css_selector("li.site-listing div.desc-container p.desc-paragraph a") for el in els: if(count < N): t = el.get_attribute('innerHTML').lower() fo = open(AD_FILE, "a") fo.write(t + '\n') fo.close() count += 1 driver.find_element_by_css_selector("a.next").click() def tearDown(self): self.vdisplay.stop() self.driver.quit()
class Webdriver(unittest.TestCase): def setUp(self): self.vdisplay = Xvfb(width=1280, height=720) self.vdisplay.start() # if(not vdisplay.start()): # fo = open(LOG_FILE, "a") # fo.write("Xvfbfailure||"+str(TREATMENTID)+"||"+str(ID)+"\n") # fo.close() # sys.exit(0) if (BROWSER == 'firefox'): if (platform.system() == 'Darwin'): self.driver = webdriver.Firefox() elif (platform.system() == 'Linux'): self.driver = webdriver.Firefox(proxy=proxy) else: print "Unidentified Platform" sys.exit(0) elif (BROWSER == 'chrome'): print "WARNING: Expecting chromedriver at specified location !!" if (platform.system() == 'Darwin'): chromedriver = "./experiment/chromedriver/chromedriver_mac" os.environ["webdriver.chrome.driver"] = chromedriver self.driver = webdriver.Chrome(executable_path=chromedriver) elif (platform.system() == 'Linux'): chromedriver = "./experiment/chromedriver/chromedriver_linux" os.environ["webdriver.chrome.driver"] = chromedriver chrome_option = webdriver.ChromeOptions() chrome_option.add_argument( "--proxy-server=yogi.pdl.cmu.edu:3128") self.driver = webdriver.Chrome(executable_path=chromedriver, chrome_options=chrome_option) else: print "Unidentified Platform" sys.exit(0) else: print "Unsupported Browser" sys.exit(0) self.driver.implicitly_wait(10) self.base_url = "https://www.google.com/" self.verificationErrors = [] self.driver.set_page_load_timeout(40) self.accept_next_alert = True def test_webdriver(self): driver = self.driver helper.setLogFile(LOG_FILE) helper.log("browserStarted||" + str(TREATMENTID), ID) run = 0 while (run < RUNS): helper.applyTreatment(driver, TREATMENTS[TREATMENTID], ID, TREATMENTID) helper.wait_for_others(AGENTS, ID, ROUND) time.sleep(20) helper.collectMeasurement(driver, MEASUREMENT, ID, TREATMENTID) run = run + 1 def tearDown(self): self.vdisplay.stop() self.driver.quit()
class PBSeleniumTest(unittest.TestCase): def setUp(self): env = os.environ self.browser_bin = env.get("BROWSER_BIN", "") # o/w use WD's default if "TRAVIS" in os.environ: self.xvfb = 1 else: # by default don't use XVFB if we are not running on CI self.xvfb = int(env.get("ENABLE_XVFB", 0)) self.pb_ext_path = self.get_extension_path() # path to the extension if self.xvfb: self.vdisplay = Xvfb(width=1280, height=720) self.vdisplay.start() self.driver = self.get_chrome_driver() self.js = self.driver.execute_script def load_url(self, url, wait_on_site=0): """Load a URL and wait before returning.""" self.driver.get(url) sleep(wait_on_site) def get_extension_path(self): """Return the path to the extension to be tested.""" if "PB_EXT_PATH" in os.environ: return os.environ["PB_EXT_PATH"] else: # check the default path if PB_EXT_PATH env. variable is empty print "Can't find the env. variable PB_EXT_PATH, will check ../.." # if the PB_EXT_PATH environment variable is not set # check the default location for the last modified crx file exts = glob("../../*.crx") # get matching files return max(exts, key=os.path.getctime) if exts else "" def txt_by_css(self, css_selector, timeout=SEL_DEFAULT_WAIT_TIMEOUT): """Find an element by CSS selector and return it's text.""" return self.find_el_by_css(css_selector, timeout).text def find_el_by_css(self, css_selector, timeout=SEL_DEFAULT_WAIT_TIMEOUT): return WebDriverWait(self.driver, timeout).until( EC.presence_of_element_located((By.CSS_SELECTOR, css_selector))) def get_chrome_driver(self): """Setup and return a Chrom[e|ium] browser for Selenium.""" opts = Options() absp = os.path.abspath if "TRAVIS" in os.environ: # github.com/travis-ci/travis-ci/issues/938 opts.add_argument("--no-sandbox") opts.add_extension(self.pb_ext_path) # will fail if ext can't be found if self.browser_bin: # otherwise will use webdriver's default binary print "Browser binary:", absp(self.browser_bin) opts.binary_location = self.browser_bin # set binary location # Fix for https://code.google.com/p/chromedriver/issues/detail?id=799 opts.add_experimental_option("excludeSwitches", ["ignore-certificate-errors"]) return webdriver.Chrome(chrome_options=opts) def tearDown(self): self.driver.quit() if self.xvfb and self.vdisplay: self.vdisplay.stop()
def get_news(conn, max_date, current_time): """ 华尔街见闻抓取 :param conn: :param max_date: 数据库中最新新闻的日期 :param current_time: 当前时间 :return: """ func_name = "采集华尔街见闻" logger.debug('start %s ' % func_name) spider_data = datetime.datetime.strptime(current_time, '%Y-%m-%d %H:%M:%S') driver = None try: xvfb = Xvfb(width=1280, height=720) xvfb.start() driver = webdriver.Firefox(executable_path=chromedriver_path) driver.get('https://wallstreetcn.com/live/global') # 让页面滚动到下面,window.scrollBy(0, scrollStep),ScrollStep :间歇滚动间距 js = 'window.scrollBy(0,3000)' driver.execute_script(js) time.sleep(5) js = 'window.scrollBy(0,60000)' driver.execute_script(js) time.sleep(5) pages = driver.page_source soup = BeautifulSoup(pages, 'html.parser') soup1 = soup.find('div', class_='livenews-main') content = soup1.find_all('div', class_='live-item') news_source = '华尔街见闻' news_type = '宏观' last_news_time = '23:59' d_date = datetime.datetime.strptime(current_time, '%Y-%m-%d %H:%M:%S') for cont in content: news_time = cont.find('time', attrs={'class': 'live-item_created'}).get_text() news = cont.find('div', attrs={'class': 'live-item_main'}).find('div', attrs={'class': 'live-item_html'}) if news is None: return news = news.get_text().strip().replace('//', '') if last_news_time < news_time: d_date = d_date - datetime.timedelta(days=1) s_date = d_date.strftime("%Y-%m-%d") over_time = s_date + ' ' + news_time if max_date > over_time: break sql_params = [over_time, spider_data, news_source, news_type, news] logger.debug(sql_cj) logger.debug(sql_params) execute_sql(conn, sql_cj, sql_params) last_news_time = news_time logger.debug('end %s ' % func_name) except Exception as e: msg = func_name + ' 处理失败: ' + str(e) logger.error(msg) finally: if driver: # driver.close() driver.quit() xvfb.stop()
def test_with_xvfb(): if use_xvfb: from xvfbwrapper import Xvfb display = Xvfb(width=1920, height=1080) display.start() my_test() if use_xvfb: display.stop()
def test_stop(self): orig_display = os.environ['DISPLAY'] xvfb = Xvfb() xvfb.start() self.assertNotEqual(orig_display, os.environ['DISPLAY']) xvfb.stop() self.assertIsNone(xvfb.proc) self.assertEqual(orig_display, os.environ['DISPLAY'])
def test_stop(self): orig_display = os.environ['DISPLAY'] xvfb = Xvfb() xvfb.start() self.assertNotEqual(orig_display, os.environ['DISPLAY']) xvfb.stop() self.assertEqual(orig_display, os.environ['DISPLAY']) self.assertIsNone(xvfb.proc)
def xvfb(self, line, cell=None): display = Xvfb(**self.xvfb_kwargs) display.start() if cell is None: self.shell.ex(line) else: self.shell.ex(cell) display.stop()
class Webdriver(unittest.TestCase): def setUp(self): self.vdisplay = Xvfb(width=1280, height=720) self.vdisplay.start() # if(not vdisplay.start()): # fo = open(LOG_FILE, "a") # fo.write("Xvfbfailure||"+str(TREATMENTID)+"||"+str(ID)+"\n") # fo.close() # sys.exit(0) if(BROWSER=='firefox'): if (platform.system()=='Darwin'): self.driver = webdriver.Firefox() elif (platform.system()=='Linux'): self.driver = webdriver.Firefox(proxy=proxy) else: print "Unidentified Platform" sys.exit(0) elif(BROWSER=='chrome'): print "WARNING: Expecting chromedriver at specified location !!" if (platform.system()=='Darwin'): chromedriver = "./experiment/chromedriver/chromedriver_mac" os.environ["webdriver.chrome.driver"] = chromedriver self.driver = webdriver.Chrome(executable_path=chromedriver) elif (platform.system() == 'Linux'): chromedriver = "./experiment/chromedriver/chromedriver_linux" os.environ["webdriver.chrome.driver"] = chromedriver chrome_option = webdriver.ChromeOptions() chrome_option.add_argument("--proxy-server=yogi.pdl.cmu.edu:3128" ) self.driver = webdriver.Chrome(executable_path=chromedriver, chrome_options=chrome_option) else: print "Unidentified Platform" sys.exit(0) else: print "Unsupported Browser" sys.exit(0) self.driver.implicitly_wait(10) self.base_url = "https://www.google.com/" self.verificationErrors = [] self.driver.set_page_load_timeout(40) self.accept_next_alert = True def test_webdriver(self): driver = self.driver cole.optIn(driver) driver.get(SITE) time.sleep(20) #raw_input("wait") pref = cole.get_ad_pref(driver) print SITE print 'pref=', pref if pref != []: fo = open(TARGET_FILE, "a") fo.write(SITE+"||"+"@".join(pref)+'\n') fo.close() def tearDown(self): self.vdisplay.stop() self.driver.quit()
class MpvTestCase(unittest.TestCase): def setUp(self): self.disp = Xvfb() self.disp.start() self.m = mpv.MPV(vo='x11') def tearDown(self): self.m.terminate() self.disp.stop()
def xvfb(request): from xvfbwrapper import Xvfb vdisplay = Xvfb() vdisplay.start() yield vdisplay.stop()
class MpvTestCase(unittest.TestCase): def setUp(self): self.disp = Xvfb() self.disp.start() self.m = mpv.MPV(vo=testvo, loglevel='debug', log_handler=timed_print()) def tearDown(self): self.m.terminate() self.disp.stop()
def test_xvfb(): '''provide xvfb in test environment''' vdisplay = Xvfb() try: # guard for multiprocessing dist test vdisplay.start() yield vdisplay vdisplay.stop() except Exception as e: yield vdisplay
class OFMTransferSiteManager(OFMSiteManager): def __init__(self, user=None): # pylint: disable=super-init-not-called self.user = user if self.user: self._login_user = self.user.ofm_username self._login_password = self.user.ofm_password else: self._login_user = os.environ('OFM_USERNAME') self._login_password = os.environ('OFM_PASSWORD') self.display = Xvfb() self.display.start() def download_transfer_excels(self, matchdays=None): profile = webdriver.FirefoxProfile(os.path.join(BASE_DIR, 'ofm_transfer_data', 'firefox_profile')) profile.set_preference("browser.download.dir", os.path.join(BASE_DIR, 'ofm_transfer_data')) self.browser = webdriver.Firefox(firefox_profile=profile) self.browser.set_page_load_timeout(10) self.login() if not matchdays: matchdays = [Matchday.get_current()] for matchday in matchdays: if not self._is_transfer_file_present(matchday): try: self._jump_to_transfer_page(self, matchday=matchday) # pylint: disable=redundant-keyword-arg except TimeoutError: pass @staticmethod def _is_transfer_file_present(matchday=None): if not matchday: matchday = Matchday.get_current() if os.path.isfile(os.path.join(BASE_DIR, 'ofm_transfer_data', 'ofm_spielerwechsel_{}_{}.csv'.format( matchday.season.number, matchday.number) )): return True return False @timeout(5, use_signals=False) def _jump_to_transfer_page(self, matchday=None): if not matchday: self.jump_to_frame(Constants.Transfer.DOWNLOAD_TRANSFERS) else: self.jump_to_frame(Constants.Transfer.DOWNLOAD_TRANSFERS_FROM_MATCHDAY.format(matchday.number)) def kill_browser(self): if self.browser: self.browser.stop_client() self.display.stop()
def test_renderer(self): vdisplay = Xvfb() vdisplay.start() ren = window.Renderer() window.record(ren, n_frames=1, out_path=self.out_file, size=(600, 600)) self.assertTrue(os.path.exists(self.out_file)) vdisplay.stop()
def cli(debug, key, xvfb, verbose, first_name, last_name, email_address, job_description, resume, job_location): """Apply to jobs automatically with Job Automate. Job Automate requires the user's first name, last name, email address, job description, and resume location prior to automating a job search. The job location is an optional argument that is left blank by default. This allows Job Automate to search for jobs across the entire United States. Each of the command line arguments requires a string data type. In order to pass a string to each argument, type each argument inside quotation marks. An example usage: jobautomate --key ABC123 "Homer" "Simpson" "*****@*****.**" "Nuclear Technician" "/path/to/resume.txt" "Springfield" When run, jobautomate will print to terminal the job position and company name tied to the easily apply application. With the --verbose flag, jobautomate will also print to terminal the applications which are not easily apply applications. """ if xvfb: vdisplay = Xvfb() vdisplay.start() user_parameters = indeed_parameters(job_description, job_location) response = access_indeed_api(user_parameters, key) if response == {'error': 'Invalid publisher number provided.'}: raise InvalidAPIKey(response) driver = webdriver.Firefox() while True: for url in retrieve_indeed_urls(response): driver.get(url) try: open_application(driver, 'indeed-apply-button') switch_frames(driver, 'iframe[name$=modal-iframe]') fill_application(driver, first_name, last_name, email_address, resume) if debug: submit_application(driver, debug=True) else: submit_application(driver) except (NoSuchElementException, ElementNotVisibleException): if verbose: print('Not an easily apply job application.') else: pass user_prompt = click.prompt( 'Would you like to continue searching for jobs? (yes/no)') if user_prompt == 'yes': user_parameters['start'] += 25 response = access_indeed_api(user_parameters, key) else: driver.quit() if xvfb: vdisplay.stop() break
class Webdriver(unittest.TestCase): def setUp(self): self.vdisplay = Xvfb(width=1280, height=720) self.vdisplay.start() # if(not vdisplay.start()): # fo = open(LOG_FILE, "a") # fo.write("Xvfbfailure||"+str(TREATMENTID)+"||"+str(ID)+"\n") # fo.close() # sys.exit(0) if(BROWSER=='firefox'): if (platform.system()=='Darwin'): self.driver = webdriver.Firefox() elif (platform.system()=='Linux'): self.driver = webdriver.Firefox(proxy=proxy) else: print "Unidentified Platform" sys.exit(0) elif(BROWSER=='chrome'): print "WARNING: Expecting chromedriver at specified location !!" if (platform.system()=='Darwin'): chromedriver = "./experiment/chromedriver/chromedriver_mac" os.environ["webdriver.chrome.driver"] = chromedriver self.driver = webdriver.Chrome(executable_path=chromedriver) elif (platform.system() == 'Linux'): chromedriver = "./experiment/chromedriver/chromedriver_linux" os.environ["webdriver.chrome.driver"] = chromedriver chrome_option = webdriver.ChromeOptions() chrome_option.add_argument("--proxy-server=yogi.pdl.cmu.edu:3128" ) self.driver = webdriver.Chrome(executable_path=chromedriver, chrome_options=chrome_option) else: print "Unidentified Platform" sys.exit(0) else: print "Unsupported Browser" sys.exit(0) self.driver.implicitly_wait(10) self.base_url = "https://www.google.com/" self.verificationErrors = [] self.driver.set_page_load_timeout(40) self.accept_next_alert = True def test_webdriver(self): driver = self.driver helper.setLogFile(LOG_FILE) helper.log("browserStarted||"+str(TREATMENTID), ID) run = 0 while (run < RUNS): helper.applyTreatment(driver, TREATMENTS[TREATMENTID], ID, TREATMENTID) helper.wait_for_others(AGENTS, ID, ROUND) time.sleep(20) helper.collectMeasurement(driver, MEASUREMENT, ID, TREATMENTID) run = run+1 def tearDown(self): self.vdisplay.stop() self.driver.quit()
def suro_ws_js_enabled_page(): vdisplay = Xvfb() vdisplay.start() browser = webdriver.Firefox() browser.get('http://www.google.com') print browser.title browser.quit() vdisplay.stop()
def html_to_png(filepaths, width=1280, height=720,save=False): """ Takes in input one or a list of paths to html files containing a cytoscape network under variable cy. Returns the base64 of the png of the network. N.B. Common possible error : - (variable cy not found) can be caused by unvalidity of the source of the cytoscape.js in the html file. :param filepaths: can be one or a list of path to html network files. absolute and relative path are supported. :param width: the image width :param height: the image height :param save: if True, the generated png will be saved in the same directory then its html counterpart. :return: a list of png encoded as base 64 unicode strings """ from selenium import webdriver from xvfbwrapper import Xvfb import os is_list = hasattr(filepaths, "__iter__") if not is_list: assert type(filepaths) is str filepaths = [filepaths] #Xvfb for headless mode xvfb = Xvfb(width=width, height=height) xvfb.start() browser = webdriver.Firefox() def cast_html_to_png(file_location): browser.get("file://"+file_location) time.sleep(1) b64 = browser.execute_script('return cy.png()') #index is always 22 index = b64.index("base64,") + len("base64,") return b64[index:] abspath = os.getcwd() png_images = [] for f in filepaths: if f[0] != '/': f = os.path.join(abspath,f) # print("File : "+f) b64image = cast_html_to_png(f) png_images.append(b64image) if save : if len(f) > 5 and f[-5:] == ".html": f = f[:-5] f += ".png" save_png(b64image,f) browser.quit() xvfb.stop() return png_images
class BaseLiveTest(LiveServerTestCase): @classmethod def setUp(self): if RUN_TESTS_IN_BACKGROUND: self.xvfb = Xvfb(width=1280, height=720) self.xvfb.start() self.browser = webdriver.Chrome() @classmethod def tearDown(self): self.browser.quit() if RUN_TESTS_IN_BACKGROUND: self.xvfb.stop() # This function performs an implicit test that the user can login. This is used in the base # test class, as the login operation is required in the majority of the test cases. def can_login(self, email='', password='', login_first_time=True): if login_first_time: # User opens the web browser and goes to RGT home page self.browser.get(self.live_server_url + '/home/') # User sees the RGT login page body = self.browser.find_element_by_tag_name('body') self.assertIn('Login', body.text) # User types email and password and hits enter email_field = self.browser.find_element_by_name('email') email_field.send_keys(email) password_field = self.browser.find_element_by_name('password') password_field.send_keys(password) password_field.send_keys(Keys.RETURN) # email and password are accepted, and the user is redirected to # the Home page body = self.browser.find_element_by_tag_name('body') self.assertIn('Welcome', body.text) def wait_for_dialog_box_with_message(self, message=''): # wait for the dialog box to appear with the desired message WebDriverWait( self.browser, 10).until(lambda x: self.browser.find_element_by_css_selector( "div[class*='ui-dialog'][style*='block']")) dialog_box = self.browser.find_element_by_class_name("ui-dialog") self.assertIn(message, dialog_box.text) # User clicks the close button to close the dialog close_dialog_button = self.browser.find_element_by_css_selector( "button[role='button']") close_dialog_button.click() # Do nothing after the dialog box disappeared time.sleep(1)
def getHtml2(url): try: vdisplay = Xvfb() vdisplay.start() browser = webdriver.Firefox() browser.set_page_load_timeout(120) browser.get(url) return browser.page_source except Exception as e: print(e) finally: vdisplay.stop()
class BaseLiveTest(LiveServerTestCase): @classmethod def setUp(self): if RUN_TESTS_IN_BACKGROUND: self.xvfb = Xvfb(width=1280, height=720) self.xvfb.start() self.browser = webdriver.Chrome() @classmethod def tearDown(self): self.browser.quit() if RUN_TESTS_IN_BACKGROUND: self.xvfb.stop() # This function performs an implicit test that the user can login. This is used in the base # test class, as the login operation is required in the majority of the test cases. def can_login(self, email='', password='', login_first_time=True): if login_first_time: # User opens the web browser and goes to RGT home page self.browser.get(self.live_server_url + '/home/') # User sees the RGT login page body = self.browser.find_element_by_tag_name('body') self.assertIn('Login', body.text) # User types email and password and hits enter email_field = self.browser.find_element_by_name('email') email_field.send_keys(email) password_field = self.browser.find_element_by_name('password') password_field.send_keys(password) password_field.send_keys(Keys.RETURN) # email and password are accepted, and the user is redirected to # the Home page body = self.browser.find_element_by_tag_name('body') self.assertIn('Welcome', body.text) def wait_for_dialog_box_with_message(self, message=''): # wait for the dialog box to appear with the desired message WebDriverWait(self.browser, 10).until( lambda x: self.browser.find_element_by_css_selector("div[class*='ui-dialog'][style*='block']")) dialog_box = self.browser.find_element_by_class_name("ui-dialog") self.assertIn(message, dialog_box.text) # User clicks the close button to close the dialog close_dialog_button = self.browser.find_element_by_css_selector("button[role='button']") close_dialog_button.click() # Do nothing after the dialog box disappeared time.sleep(1)
def screengrab_firefox(url): """get screenshot: firefox with adblock and virtual display""" vdisplay = Xvfb() vdisplay.start() fp = webdriver.FirefoxProfile() fp.add_extension('adblockplusfirefox.xpi') fp.set_preference("extensions.adblockplus.currentVersion", "2.4") fox = webdriver.Firefox(fp) fox.get(url) fox.save_screenshot('ToIBot.png') fox.quit() vdisplay.stop()
class PBSeleniumTest(unittest.TestCase): def setUp(self): env = os.environ self.browser_bin = env.get("BROWSER_BIN", "") # o/w use WD's default self.xvfb = int(env.get("ENABLE_XVFB", 1)) # enabled by default self.pb_ext_path = self.get_extension_path() # path to the extension if self.xvfb: self.vdisplay = Xvfb(width=1280, height=720) self.vdisplay.start() self.driver = self.get_chrome_driver() self.driver.implicitly_wait(10) self.js = self.driver.execute_script def get_extension_path(self): """Return the path to the extension to be tested.""" if "PB_EXT_PATH" in os.environ: return os.environ["PB_EXT_PATH"] else: # check the default path if PB_EXT_PATH env. variable is empty print "Can't find the environment variable PB_EXT_PATH" # if the PB_EXT_PATH environment variable is not set # check the default location for the last modified crx file exts = glob("../../*.crx") # get matching files return max(exts, key=os.path.getctime) if exts else "" def txt_by_css(self, css_selector): """Find an element by CSS selector and return it's text.""" return self.driver.find_element_by_css_selector(css_selector).text def get_chrome_driver(self): """Setup and return a Chrom[e|ium] browser for Selenium.""" opts = Options() absp = os.path.abspath if "TRAVIS" in os.environ: # github.com/travis-ci/travis-ci/issues/938 opts.add_argument("--no-sandbox") opts.add_extension(self.pb_ext_path) # will fail if ext can't be found if self.browser_bin: # otherwise will use webdriver's default binary print "Browser binary:", absp(self.browser_bin) opts.binary_location = self.browser_bin # set binary location # Fix for https://code.google.com/p/chromedriver/issues/detail?id=799 opts.add_experimental_option("excludeSwitches", ["ignore-certificate-errors"]) return webdriver.Chrome(chrome_options=opts) def tearDown(self): self.driver.quit() if self.xvfb and self.vdisplay: self.vdisplay.stop()
class Headless(Browser): def __init__(self): from xvfbwrapper import Xvfb self.width = runtime.settings.SELENIUM_ARGS.get('WINDOW_SIZE_WIDTH') if self.width == False: self.width = 1024 self.height = runtime.settings.SELENIUM_ARGS.get('WINDOW_SIZE_HEIGHT') if self.height == False: self.height = 768 self._vdisplay = Xvfb(width=self.width,height=self.height) self._vdisplay.start() super(Headless, self).__init__() super(Headless, self).__init__() def clear(self): super(Headless, self).clear() self._vdisplay.stop()
class Surfer: """This class does virtual web surfing on our demand """ def __init__(self,delay=2): self.extraDelay = delay #extra time to wait after each operation (s) from selenium import webdriver from selenium.webdriver.support.ui import WebDriverWait # available since 2.4.0 from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as ExpectedConditions from xvfbwrapper import Xvfb self.vdisplay = Xvfb() self.vdisplay.start() self.selenium_profile = webdriver.FirefoxProfile() self.selenium_driver = webdriver.Firefox(self.selenium_profile) def surfTo(self,url): self.selenium_driver.get(url) self.selenium_driver.implicitly_wait(self.extraDelay) def clickOnStuff(self,xPath): elementList = self.selenium_driver.find_elements_by_xpath(xPath) if not elementList: logging.warning("No elements found for xPath `%s`" % xPath) else: for element in elementList: element.click() self.selenium_driver.implicitly_wait(self.extraDelay) def findElements(self,xPath): return self.selenium_driver.find_elements_by_xpath(xPath) def getUrlList(self,xPath): urlList = [] elementList = self.findElements(xPath) for element in elementList: href = element.get_attribute("href") if href is not None: urlList.append(Url(href)) return urlList#list of Url objects def kill(self): self.selenium_driver.close() self.vdisplay.stop()
class TestHomepages(unittest.TestCase): def setUp(self): self.vdisplay = Xvfb(width=800, height=600) self.vdisplay.start() self.browser = webdriver.Firefox() def testUbuntuHomepage(self): self.browser.get('http://www.ubuntu.com') self.assertIn('Ubuntu', self.browser.title) def testGoogleHomepage(self): self.browser.get('http://www.google.com') self.assertIn('Google', self.browser.title) def tearDown(self): self.browser.quit() self.vdisplay.stop()
def getAuth(): import requests import settings from time import sleep from splinter import Browser from xvfbwrapper import Xvfb vdisplay = Xvfb(width=3840, height=2160) vdisplay.start() client_id = settings.getClientID() email = settings.getEmail() password = settings.getPassword() team_name = settings.getTeamName() scopes = ('users:read', 'channels:write', 'channels:history', 'channels:read', 'chat:write:bot', 'chat:write:bot', 'chat:write:bot', 'dnd:write', 'dnd:read', 'emoji:read', 'files:write:user', 'files:read', 'groups:write', 'groups:history', 'groups:read', 'im:write', 'im:history', 'im:read', 'mpim:write', 'mpim:history', 'mpim:read', 'pins:write', 'pins:read', 'reactions:write', 'reactions:read', 'reminders:write', 'reminders:read', 'search:read', 'stars:write', 'stars:read', 'team:read', 'usergroups:write', 'usergroups:read', 'users:write', 'admin') scopeStr = '+'.join(scopes) browser = Browser('chrome') url = 'https://slack.com/oauth/authorize?client_id={}&scope={}&state=test'.format(client_id, scopeStr) browser.visit(url) browser.fill('domain', team_name) browser.find_by_id('submit_team_domain').first.click() browser.fill('email', email) browser.fill('password', password) browser.find_by_id('signin_btn').first.click() button = browser.find_by_id('oauth_authorizify') button.mouse_over() button.click() codeStart = browser.url.find('=') + 1 codeEnd = browser.url.find('&') code = browser.url[codeStart:codeEnd] browser.quit() vdisplay.stop() return code
def cli(debug, key, xvfb, verbose, first_name, last_name, email_address, job_description, resume, job_location): """Apply to jobs automatically with Job Automate. Job Automate requires the user's first name, last name, email address, job description, and resume location prior to automating a job search. The job location is an optional argument that is left blank by default. This allows Job Automate to search for jobs across the entire United States. Each of the command line arguments requires a string data type. In order to pass a string to each argument, type each argument inside quotation marks. An example usage: jobautomate "Homer" "Simpson" "*****@*****.**" "Nuclear Technician" "/path/to/resume.txt" "Springfield" When run, jobautomate will print to terminal the job position and company name tied to the easily apply application. With the --verbose flag, jobautomate will also print to terminal the applications which are not easily apply applications. """ if xvfb: vdisplay = Xvfb() vdisplay.start() driver = webdriver.Firefox() user_parameters = indeed_parameters(job_description, job_location) while True: for url in indeed_urls(user_parameters, key): driver.get(url) try: find_apply_button(driver, "indeed-apply-button") switch_frames(driver, "iframe[name$=modal-iframe]") fill_application(driver, first_name, last_name, email_address, resume) if debug: apply_or_continue(driver, debug=True) except (NoSuchElementException, ElementNotVisibleException): if verbose: print("Not an easily apply job application.") else: pass user_prompt = click.prompt("Would you like to continue searching for jobs? (yes/no") if user_prompt == "yes": user_parameters["start"] += 25 else: driver.quit() if xvfb: vdisplay.stop() break
def return_html_code(url, proxy_use): vdisplay = Xvfb() vdisplay.start() proxy_address_list = get_proxy_fastest() if proxy_address_list != False: proxy_address = random.choice(proxy_address_list) ip, port = proxy_address.split(":") print ip, port profile = webdriver.FirefoxProfile() profile.set_preference("network.proxy.http", ip) profile.set_preference("network.proxy.http_port", port) profile.set_preference("network.proxy_type", 1) driver = webdriver.Firefox(firefox_profile=profile) else: print "Using localhost, unable to get proxy" driver = webdriver.Firefox() driver.maximize_window() driver.get(url) # initial wait for the tweets to load # initial wait for the tweets to load wait = WebDriverWait(driver, 30) try: wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "li[data-item-id]"))) except TimeoutException: driver.quit() return False # scroll down to the last tweet until there is no more tweets loaded while True: tweets = driver.find_elements_by_css_selector("li[data-item-id]") print len(tweets) # added in edit 1 number_of_tweets = len(tweets) driver.execute_script("arguments[0].scrollIntoView(true);", tweets[-1]) try: wait.until( wait_for_more_than_n_elements_to_be_present((By.CSS_SELECTOR, "li[data-item-id]"), number_of_tweets) ) except TimeoutException: break html_full_source = driver.page_source driver.close() vdisplay.stop() return html_full_source
def get_dynamic_html(url): """ Function to get dynamic html of given url """ if not dynamic_dependencies: print "The extra depencies to scrape aren't"\ " installed\n Please make sure you install them\n"\ "(selenium , xvfbwrapper for python are the "\ "extra dependencies)" else: source = '' virtual_display = Xvfb() virtual_display.start() # Start virtual display browser = webdriver.Firefox() print "Finding the given page...." browser.get(url) source = browser.page_source virtual_display.stop() return source
class Screenshot(object): def __init__(self, output, size=(1200, 600), wait=1000): self.output = output self.size = size self.wait = wait def __enter__(self): self.vdisplay = Xvfb(width=self.size[0], height=self.size[1]) self.vdisplay.start() self.app = wx.App(False) wx.UIActionSimulator().MouseMove(0, 0) return self def set_window(self, window): self.window = window def __exit__(self, type, value, traceback): def _screenshot(): rect = self.window.GetRect() if sys.platform == 'linux2': client_x, client_y = self.window.ClientToScreen((0, 0)) border_width = client_x - rect.x title_bar_height = client_y - rect.y rect.width += (border_width * 2) rect.height += title_bar_height + border_width dcScreen = wx.ScreenDC() bmp = wx.EmptyBitmap(rect.width, rect.height) memDC = wx.MemoryDC() memDC.SelectObject(bmp) memDC.Blit(0, 0, rect.width, rect.height, dcScreen, rect.x, rect.y) memDC.SelectObject(wx.NullBitmap) img = bmp.ConvertToImage() img.SaveFile(self.output, wx.BITMAP_TYPE_PNG) wx.FutureCall(self.wait, _screenshot) wx.FutureCall(self.wait + 1000, self.window.Destroy) self.app.MainLoop() self.vdisplay.stop()
def startUser(name, url, username, password, headless, logDir, chromedriver, workHour): if headless: xvfb = Xvfb(width=1100, height=800) xvfb.start() user = User(name, url=url, username=username, password=password, logDir=logDir, chromedriver=chromedriver, workHour=workHour) # TODO - configure workflow # Always start with Login() and end with Logout(). There has to be at least # one workflow between Login() and Logout(). user.addWorkflow([ LogInOutWorkflow.Login(), MonitorDashboard(), MonitorEvents(), MonitorDashboard(), MonitorEvents(), MonitorDashboard(), MonitorEvents(), MonitorDashboard(), MonitorEvents(), LogInOutWorkflow.Logout()]) try: user.work() except: user.log("%s unexpectedly failed" % user.name) traceback.print_exc() finally: #log results resultsStr = "" for result in user.results: resultsStr += str(result) resultsStr += "," user.log(resultsStr) print "cleaning up %s" % user.name user.quit() if headless: xvfb.stop()
class FrontendTestCase(LiveServerTestCase): """ Test frontend access through firefox webdriver """ def create_app(self): return app def setUp(self): self.xvfb = Xvfb(width=1024, height=768) self.xvfb.start() self.browser = webdriver.Firefox() def tearDown(self): self.browser.quit() self.xvfb.stop() def test_index_access(self): """ Try to Access the first page. :return: """ url = self.get_server_url() self.browser.get(url) # Can we see the right value on title bar? self.assertIn('Home', self.browser.title) # Can we find a specific page? self.browser.find_element_by_link_text('Find Page').click() self.assertIn('find-page', self.browser.current_url) self.fail('Implement Find Page.') self.browser.find_element_by_id('search-field').text('index') self.browser.find_element_by_link_text('Search').click() self.browser.find_elements_by_link_text('Index')