def jupyter_play(env, agent, steps=1000):
from IPython import display
policy = agent.get_policy()
vdisplay = Xvfb(width=1280, height=740)
vdisplay.start()
observation = env.reset()
def showarray(a, fmt='png'):
a = np.uint8(a)
f = io.BytesIO()
ima = PIL.Image.fromarray(a).save(f, fmt)
return f.getvalue()
imagehandle = display.display(display.Image(data=showarray(env.render(mode='rgb_array')), width=450), display_id='gymscr')
action = 0
reward = 0
for _ in range(steps):
time.sleep(0.001)
action = agent.compute_action(observation, prev_reward=reward, prev_action=action)
observation, reward, done, info = env.step(action)
display.update_display(display.Image(data=showarray(env.render(mode='rgb_array')), width=450), display_id='gymscr')
if done: break
vdisplay.stop()
def get_seller_phone(url):
# телефон показывается в виде картинки, используем selenium и pytesseract
vdisplay = Xvfb()
vdisplay.start()
driver = webdriver.Chrome(options=options)
driver.set_window_size(1920, 1080)
driver.get(url)
try:
button = driver.find_element_by_xpath('//a[@class="button item-phone-button js-item-phone-button '
'button-origin button-origin-blue button-origin_full-width '
'button-origin_large-extra item-phone-button_hide-phone '
'item-phone-button_card js-item-phone-button_card"]')
button.click()
time.sleep(2)
driver.save_screenshot("phone_number.png")
image = driver.find_element_by_xpath('//div[@class="item-phone-big-number js-item-phone-big-number"]//*')
cropped = Image.open("phone_number.png")
x, y = image.location["x"], image.location["y"]
width, height = image.size["width"], image.size["height"]
cropped.crop((x, y, x + width, y + height)).save("phone.gif")
phone = Image.open("phone.gif")
phone_text = image_to_string(phone)
except Exception as e:
with open("logs.txt", "a", encoding="utf8") as file:
file.write(str(e) + " avito get_seller_phone\n")
phone_text = "Не указано"
driver.quit()
vdisplay.stop()
return phone_text
def runner(context):
driverManager = context['driverManager']
user = context['user']
console: Console = context['console']
playlist = context['playlist']
queueUrl = context['queueUrl']
proxy = context['proxy']
shutdownEvent = context['shutdownEvent']
pid = current_process().pid
try:
vdisplay = Xvfb(width=1280,
height=1024,
colordepth=24,
tempdir=None,
noreset='+render')
vdisplay.start()
driverData = driverManager.getDriver(type='chrome',
uid=pid,
user=user,
proxy=proxy)
if not driverData:
return
driver = driverData['driver']
userDataDir = driverData['userDataDir']
if not driver:
return
except:
console.error('Unavailale webdriver: %s' % format_exc())
else:
try:
spotify = Spotify.Adapter(driver, console, shutdownEvent)
if not shutdownEvent.is_set():
if spotify.register(user):
message = {'user': user, 'playlist': playlist}
client = client('sqs')
client.send_message(
QueueUrl=queueUrl,
MessageBody=dumps(message),
DelaySeconds=1,
)
except:
console.exception()
if driver:
try:
driver.quit()
except:
pass
if userDataDir:
try:
rmtree(path=userDataDir, ignore_errors=True)
except:
pass
if vdisplay:
try:
vdisplay.stop()
except:
pass
def handle(self, *args, **options):
xvfb = Xvfb(width=1600, height=720)
xvfb.start()
browser = WebDriver()
browser.get(settings.DOMAIN)
sleep(1)
browser.find_element_by_css_selector("#disclaimer button").click()
sleep(0.7)
browser.find_elements_by_css_selector(".officer .checkmark")[4].click()
sleep(2)
browser.find_element_by_css_selector(
".complaint-row .col-md-3").click()
sleep(1)
content = browser.get_screenshot_as_png()
now = datetime.datetime.now().strftime("%Y-%m-%d %H-%M-%S")
file_name = "{now}.png".format(now=now)
file_path = os.path.join(settings.BASE_DIR, 'static', file_name)
with open(file_path, "wb") as f:
f.write(content)
browser.quit()
email = EmailMessage(subject='CPDB Homepage screenshot %s' % now,
body='FYI',
to=['*****@*****.**'])
email.attach_file(file_path)
email.send()
xvfb.stop()
class Test(unittest.TestCase):
def setUp(self):
self.xvfb = os.environ.get("ENABLE_XVFB", False)
self.browser = os.environ.get("BROWSER", "Chrome")
if self.xvfb:
self.vdisplay = Xvfb(width=1280, height=720)
self.vdisplay.start()
if self.browser == "Firefox":
self.driver = self.get_ff_driver()
else:
self.driver = self.get_chrome_driver()
self.load = self.driver.get
def tearDown(self):
if self.driver:
self.driver.quit()
if self.xvfb and self.vdisplay:
self.vdisplay.stop()
def get_ff_driver(self):
return webdriver.Firefox()
def get_chrome_driver(self):
opts = Options()
if "TRAVIS" in os.environ: # github.com/travis-ci/travis-ci/issues/938
opts.add_argument("--no-sandbox")
# Fix for https://code.google.com/p/chromedriver/issues/detail?id=799
opts.add_experimental_option("excludeSwitches",
["ignore-certificate-errors"])
return webdriver.Chrome(chrome_options=opts)
def test_should_load_wikipedia(self):
self.load("https://www.wikipedia.org")
self.assertIn("Wikipedia", self.driver.title)
def get_apartment_data(url):
vdisplay = Xvfb()
vdisplay.start()
driver = webdriver.Chrome(options=options)
driver.set_window_size(1920, 1080)
driver.get(url)
city, district, street, block_number = get_address(driver)
sell_type, rent_info = get_selling_type(url)
if "продажа" in sell_type.lower():
rent_info = "Не аренда"
material, lift, year, rooms_number, floor, total_floors, total_area, kitchen_area, repair = get_apartment_params(driver)
block_type = "Вторичка"
living_area = "Не указано"
price = get_price(driver)
if "Аренда" in sell_type:
if "posutochno" in url:
price += "/день"
else:
price += "/мес."
#seller_type, seller_name = get_seller_info(driver)
images = get_photos(driver)
description = get_description(driver)
phone = get_seller_phone(driver)
selling_detail = "Не указано"
driver.quit()
vdisplay.stop()
return [city, district, street, block_number, sell_type, rent_info, price, block_type,
rooms_number, total_area, total_floors, material, selling_detail, images,
description, phone, kitchen_area, living_area, floor]
def main():
import sys
import imageio
import pandas as pd
import matplotlib.pyplot as plt
from xvfbwrapper import Xvfb
plt.ioff()
# with Xvfb() as xvfb:
# plt.ioff()
vdisplay = Xvfb()
vdisplay.start()
if len(sys.argv)<5:
subsampleRate = 10
else:
subsampleRate = int(sys.argv[4])
if len(sys.argv)<6:
speedup = 10
else:
speedup = int(sys.argv[5])
bounds2video(sys.argv[1],sys.argv[2],sys.argv[3],subsampleRate,speedup)
vdisplay.stop()
def main():
import sys
import imageio
import pandas as pd
import matplotlib.pyplot as plt
from xvfbwrapper import Xvfb
plt.ioff()
# with Xvfb() as xvfb:
# plt.ioff()
vdisplay = Xvfb()
vdisplay.start()
if len(sys.argv) < 5:
subsampleRate = 10
else:
subsampleRate = int(sys.argv[4])
if len(sys.argv) < 6:
speedup = 10
else:
speedup = int(sys.argv[5])
bounds2video(sys.argv[1], sys.argv[2], sys.argv[3], subsampleRate, speedup)
vdisplay.stop()
def tree_builder(node_file, function_folder):
"""
Perform tree builder script to extract csv, tsv and svg.
To correctly run tree_builder needs Xorg. We are using xvfb-run to workaround
on "x-server-less" systems.
Script usage: python new_tree_builder_for_perl_tango.py -d visualization_ITSoneDB_r131.dmp
"""
sys.stdout.write('\n### Step 5: Tree Builder\n')
# xvbf-run issue:
# /bin/xvfb-run: line 181: 17066 Segmentation fault DISPLAY=:$SERVERNUM XAUTHORITY=$AUTHFILE "$@" 2>&1
# solution: https://unix.stackexchange.com/questions/152957/xvfb-run-aborting-on-cluster
###command = 'xvfb-run -d -e /tmp/biomas_xvfb.log '+__python_executable__+' '+__location__+'/new_tree_builder_for_perl_tango.py -d '+ visualization
###command = 'xvfb-run '+__python_executable__+' '+__location__+'/new_tree_builder_for_perl_tango.py -d '+ visualization
#stdout, stderr, status = run_command(command)
command = __python_executable__+' '+__location__+'/new_tree_builder_for_perl_tango.py -d '+ node_file + ' -F ' + function_folder
sys.stdout.write(command)
from xvfbwrapper import Xvfb
vdisplay = Xvfb()
vdisplay.start()
stdout, stderr, status = run_command(command)
vdisplay.stop()
# Read stdout for errors
if status == 0:
sys.stdout.write(stdout)
else:
sys.stderr.write(stderr)
class BaseSlackParser:
"""Class for basic slack parser functionality. """
def __init__(self,
base_addr,
browser="firefox",
executable_path="./bot/core/drivers/geckodriver",
browser_window_size=(1920, 1080),
page_load_timeout=30,
sticky_timeout=30,
headless=True):
self._base_addr = base_addr
self._headless = headless
if self._headless:
self.xvfb = Xvfb()
self.xvfb.start()
self.browser = Browser(browser,
headless=False,
wait_time=30,
executable_path=executable_path)
self.browser.driver.implicitly_wait(sticky_timeout)
self.browser.driver.set_page_load_timeout(page_load_timeout)
self.browser.driver.set_window_size(*browser_window_size)
self.browser.visit(self._base_addr)
def __del__(self):
if self._headless:
self.xvfb.stop()
self.browser.quit()
class BaseTestCase(unittest.TestCase):
def setUp(self):
self.xvfb = os.environ.get("ENABLE_XVFB", False)
self.browser = os.environ.get("BROWSER", "Chrome")
if self.xvfb:
self.vdisplay = Xvfb(width=1280, height=720)
self.vdisplay.start()
if self.browser == "Firefox":
self.driver = self.get_ff_driver()
else:
self.driver = self.get_chrome_driver()
self.load = self.driver.get
def tearDown(self):
if self.driver:
self.driver.quit()
if self.xvfb and self.vdisplay:
self.vdisplay.stop()
def get_ff_driver(self):
return webdriver.Firefox()
def get_chrome_driver(self):
opts = Options()
if "TRAVIS" in os.environ: # github.com/travis-ci/travis-ci/issues/938
opts.add_argument("--no-sandbox")
# Fix for https://code.google.com/p/chromedriver/issues/detail?id=799
opts.add_experimental_option("excludeSwitches",
["ignore-certificate-errors"])
return webdriver.Chrome(chrome_options=opts)
class TestJavascript(unittest.TestCase):
def setUp(self):
# Configure a web service.
handler = http.server.SimpleHTTPRequestHandler
address = ("127.0.0.1", 9999)
socketserver.TCPServer.allow_reuse_address = True # Prevents address conflicts.
httpd = socketserver.TCPServer(address, handler)
# Start the web service in a separate thread as deamon.
httpd_thread = threading.Thread(target=httpd.serve_forever)
httpd_thread.setDaemon(True)
httpd_thread.start()
# Start a display.
self.display = Xvfb()
self.display.start()
# Start the browser driver for selenium testing.
self.driver = webdriver.Firefox()
self.driver.get("http://localhost:9999/index.html")
def test_javascript(self):
# Create an instance of the selenium Firefox driver.
error = self.driver.find_elements_by_id("error")[0].text
self.assertEqual(error, "")
def tearDown(self):
self.display.stop()
def run():
#driver = webdriver.Chrome(chrome_options=options)
print('Sreencast website animation')
xvfb = Xvfb(width=1280, height=720, colordepth=24)
xvfb.start()
options = webdriver.chrome.options.Options()
options.add_argument("--no-sandbox")
options.add_argument("--disable-setuid-sandbox")
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option('useAutomationExtension', False)
options.add_argument("--kiosk")
options.add_argument("--disable-extensions")
browser = webdriver.Chrome(chrome_options=options)
#browser = webdriver.Firefox()
url = 'https://clickcall.co.nz/af939c60-79bd-4d92-9785-e58e3944bd4f'
#destination = 'movie.flv'
browser.get(url)
# normal quality, lagging in the first part on the video. filesize ~7MB
# ffmpeg_stream = 'ffmpeg -f x11grab -s 1280x720 -r 24 -i :%d+nomouse -c:v libx264 -preset superfast -pix_fmt yuv420p -s 1280x720 -threads 0 -f flv "%s"' % (xvfb.new_display, destination)
# high quality, no lagging but huge file size ~50MB
ffmpeg_stream = 'ffmpeg -y -r 30 -f x11grab -s 1280x720 -probesize 42M -i :%d+nomouse -c:v libx264rgb -crf 15 -preset:v ultrafast -c:a pcm_s16le -af aresample=async=1:first_pts=0 /output/out.mkv' % xvfb.new_display
#ffmpeg_stream = 'ffmpeg -f x11grab -s 1920x1080 -r 30 -i :0.0 -q:v 0 -c:v libx264rgb -crf fast -y /output/screen.mp4' % xvfb.new_display
args = shlex.split(ffmpeg_stream)
p = subprocess.Popen(args)
print("-------------------------------")
print(p)
print("-------------------------------")
time.sleep(30) # record for 30 secs
browser.quit()
xvfb.stop()
class OFMTransferSiteManager(OFMSiteManager):
def __init__(self, user=None): # pylint: disable=super-init-not-called
self.user = user
if self.user:
self._login_user = self.user.ofm_username
self._login_password = self.user.ofm_password
else:
self._login_user = os.environ('OFM_USERNAME')
self._login_password = os.environ('OFM_PASSWORD')
self.display = Xvfb()
self.display.start()
def download_transfer_excels(self, matchdays=None):
profile = webdriver.FirefoxProfile(
os.path.join(BASE_DIR, 'ofm_transfer_data', 'firefox_profile'))
profile.set_preference("browser.download.dir",
os.path.join(BASE_DIR, 'ofm_transfer_data'))
self.browser = webdriver.Firefox(firefox_profile=profile)
self.browser.set_page_load_timeout(10)
self.login()
if not matchdays:
matchdays = [Matchday.get_current()]
for matchday in matchdays:
if not self._is_transfer_file_present(matchday):
try:
self._jump_to_transfer_page(self, matchday=matchday) # pylint: disable=redundant-keyword-arg
except TimeoutError:
pass
@staticmethod
def _is_transfer_file_present(matchday=None):
if not matchday:
matchday = Matchday.get_current()
if os.path.isfile(
os.path.join(
BASE_DIR, 'ofm_transfer_data',
'ofm_spielerwechsel_{}_{}.csv'.format(
matchday.season.number, matchday.number))):
return True
return False
@timeout(5, use_signals=False)
def _jump_to_transfer_page(self, matchday=None):
if not matchday:
self.jump_to_frame(Constants.Transfer.DOWNLOAD_TRANSFERS)
else:
self.jump_to_frame(
Constants.Transfer.DOWNLOAD_TRANSFERS_FROM_MATCHDAY.format(
matchday.number))
def kill_browser(self):
if self.browser:
self.browser.stop_client()
self.display.stop()
class Webdriver(unittest.TestCase):
def setUp(self):
self.vdisplay = Xvfb(width=1280, height=720)
self.vdisplay.start()
# if(not vdisplay.start()):
# fo = open(LOG_FILE, "a")
# fo.write("Xvfbfailure||"+str(TREATMENTID)+"||"+str(ID)+"\n")
# fo.close()
# sys.exit(0)
if (BROWSER == 'firefox'):
if (platform.system() == 'Darwin'):
self.driver = webdriver.Firefox()
elif (platform.system() == 'Linux'):
self.driver = webdriver.Firefox(proxy=proxy)
else:
print "Unidentified Platform"
sys.exit(0)
elif (BROWSER == 'chrome'):
print "WARNING: Expecting chromedriver at specified location !!"
if (platform.system() == 'Darwin'):
chromedriver = "./experiment/chromedriver/chromedriver_mac"
os.environ["webdriver.chrome.driver"] = chromedriver
self.driver = webdriver.Chrome(executable_path=chromedriver)
elif (platform.system() == 'Linux'):
chromedriver = "./experiment/chromedriver/chromedriver_linux"
os.environ["webdriver.chrome.driver"] = chromedriver
chrome_option = webdriver.ChromeOptions()
chrome_option.add_argument(
"--proxy-server=yogi.pdl.cmu.edu:3128")
self.driver = webdriver.Chrome(executable_path=chromedriver,
chrome_options=chrome_option)
else:
print "Unidentified Platform"
sys.exit(0)
else:
print "Unsupported Browser"
sys.exit(0)
self.driver.implicitly_wait(10)
self.base_url = "https://www.google.com/"
self.verificationErrors = []
self.driver.set_page_load_timeout(40)
self.accept_next_alert = True
def test_webdriver(self):
driver = self.driver
cole.optIn(driver)
driver.get(SITE)
time.sleep(20)
#raw_input("wait")
pref = cole.get_ad_pref(driver)
print SITE
print 'pref=', pref
if pref != []:
fo = open(TARGET_FILE, "a")
fo.write(SITE + "||" + "@".join(pref) + '\n')
fo.close()
def tearDown(self):
self.vdisplay.stop()
self.driver.quit()
def work():
while True:
try:
clone = requests.get("{}/api/clone".format(url)).json()
print(clone)
vdisplay = Xvfb()
vdisplay.start()
display = Display(visible=0, size=(800, 600))
display.start()
driver = helper._init(clone['ip'], clone['port'], clone['c_user'],
clone['xs'])
try:
helper.newest_message(driver)
helper.request_message(driver)
#active_user_links = helper.active_conversations(driver)
#helper.message_to_active_users(driver, active_user_links)
except Exception as e:
print("Ex 1 : {}".format(e))
driver.save_screenshot('log-{}.{}'.format(clone['c_user'], 'png'))
driver.quit()
display.stop()
vdisplay.stop()
except Exception as e:
print(e)
time.sleep(5)
def plot_response(response_src, out_png=False):
# start virtual display
print("starting Xvfb");
vdisplay = Xvfb()
vdisplay.start()
response_src = np.loadtxt(src_txt)
if len(response_src.shape) > 1:
response_src = response_src[1]
sphere = get_sphere('symmetric724')
sh_resp = AxSymShResponse(0, response_src)
sig_resp = sh_resp.on_sphere(sphere)
sig_resp = sig_resp[None, None, None, :]
ren = window.Renderer()
sphere_actor = actor.odf_slicer(sig_resp, sphere=sphere,colormap='blues')
ren.add(sphere_actor)
my_camera = ren.camera()
my_camera.SetPosition(1.62, -9.19, 4.01)
my_camera.SetFocalPoint(0.01, -0.46, -0.19)
my_camera.SetViewUp(0.24, 0.46, 0.86)
if out_png != False:
window.record(ren, out_path=out_png, magnification=10, size=(60, 60))
else:
window.show(ren, reset_camera=False)
print('Camera Settings')
print('Position: ', '(%.2f, %.2f, %.2f)' % my_camera.GetPosition())
print('Focal Point: ', '(%.2f, %.2f, %.2f)' % my_camera.GetFocalPoint())
print('View Up: ', '(%.2f, %.2f, %.2f)' % my_camera.GetViewUp())
vdisplay.stop()
def enable():
import time
import requests
import settings
from splinter import Browser
from xvfbwrapper import Xvfb
print "Trying to enable myself."
vdisplay = Xvfb()
vdisplay.start()
email = settings.getEmail()
password = settings.getPassword()
team_name = settings.getTeamName()
bot_user = settings.getBotUser()
browser = Browser('chrome')
url = 'https://{}.slack.com/services/{}'.format(team_name, bot_user)
browser.visit(url)
browser.fill('email', email)
browser.fill('password', password)
browser.find_by_id('signin_btn').first.click()
browser.find_link_by_text('Enable').first.click()
time.sleep(2) # Sometimes I saw a crash where there was no alert, so we'll wait a bit first.
alert = browser.get_alert()
alert.accept()
time.sleep(2) # If you close the display too quickly, the request doesn't get processed.
vdisplay.stop()
def worker(game, play_game, solution_q, reward_q, stop, show_screen):
""" A worker that plays the game with a proposed solution.
Workers fetch proposed solutions from solution_q (Solution Queue) and put the
matching reward into the reward_q (Reward Queue). If a stop signal is received the
worker terminates. Rewards are matched to solutions at the receiving and by using
the solution_id as an identifier.
"""
if not show_screen:
vdisplay = Xvfb()
vdisplay.start()
while not stop.is_set():
try:
(solution_id, solution) = solution_q.get(False, 0.1)
except queue.Empty:
pass
else:
name = multiprocessing.current_process().name
logger.info("%s started working on %s", name, solution_id)
reward = play_game(solution=solution)
reward_q.put((solution_id, reward))
if not show_screen:
vdisplay.stop()
def tree_builder(visualization):
"""
Perform tree builder script to extract csv, tsv and svg.
To correctly run tree_builder needs Xorg. We are using xvfb-run to workaround
on "x-server-less" systems.
Script usage: python new_tree_builder_for_perl_tango.py -d visualization_ITSoneDB_r131.dmp
"""
sys.stdout.write('\n### Step 5: Tree Builder\n')
#command = 'xvfb-run -a -e /export/xvfb.log '+__python_executable__+' '+__location__+'/new_tree_builder_for_perl_tango.py -d '+ visualization
#stdout, stderr, status = run_command(command)
command = __python_executable__+' '+__location__+'/new_tree_builder_for_perl_tango.py -d '+ visualization
from xvfbwrapper import Xvfb
vdisplay = Xvfb()
vdisplay.start()
stdout, stderr, status = run_command(command)
vdisplay.stop()
# Read stdout for errors
if status == 0:
sys.stdout.write(stdout)
else:
sys.stderr.write(stdout)
sys.stderr.write(stderr)
class Xvfb(Plugin):
def options(self, parser, env):
super(Xvfb, self).options(parser, env)
parser.add_option("--with-xvfb-options", action="store",
dest="xvfb_options",
default=env.get("NOSE_WITH_XVFB_OPTIONS"),
help="Options to pass to Xvfb. Comma delimited with "
"equals as separators if necessary. "
"E.g. \"extension=SELINUX, once\". Currently, there is no "
"way to provide options that begin with a +. This is a limitation "
"in xvfbwrapper. Repetition is not allowed. [NOSE_WITH_XVFB_OPTIONS]")
def configure(self, options, noseconfig):
super(Xvfb, self).configure(options, noseconfig)
self.xvfb_options = {}
if options.xvfb_options:
opts = [x.strip() for x in options.xvfb_options.split(",")]
for item in opts:
key, sign, value = item.partition("=")
if not value:
value = ''
self.xvfb_options[key] = value
def begin(self):
logger.info('Starting xvfb virtual display 1024x768 with %s' % self.xvfb_options)
self.vdisplay = XvfbWrapper(width=1024, height=768, **self.xvfb_options)
self.vdisplay.start()
def finalize(self, result):
logger.info('Stopping xvfb virtual display')
self.vdisplay.stop()
def get_page(url):
path_to_chromedriver = '/usr/bin/chromedriver' # change path as needed
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--no-sandbox')
vdisplay = Xvfb()
vdisplay.start()
display = Display(visible=0, size=(1024, 768))
display.start()
browser = webdriver.Chrome(
executable_path=path_to_chromedriver,
service_args=["--verbose", "--log-path=/tmp/CHROMIUM_LOG"],
options=chrome_options)
browser.get(url)
time.sleep(2)
page = browser.page_source
browser.close()
display.stop()
vdisplay.stop()
return page
class Webdriver(unittest.TestCase):
def setUp(self):
self.vdisplay = Xvfb(width=1280, height=720)
self.vdisplay.start()
# if(not vdisplay.start()):
# fo = open(LOG_FILE, "a")
# fo.write("Xvfbfailure||"+str(TREATMENTID)+"||"+str(ID)+"\n")
# fo.close()
# sys.exit(0)
if(BROWSER=='firefox'):
if (platform.system()=='Darwin'):
self.driver = webdriver.Firefox()
elif (platform.system()=='Linux'):
self.driver = webdriver.Firefox(proxy=proxy)
else:
print "Unidentified Platform"
sys.exit(0)
elif(BROWSER=='chrome'):
if (platform.system()=='Darwin'):
chromedriver = "./experiment/chromedriver/chromedriver_mac"
os.environ["webdriver.chrome.driver"] = chromedriver
self.driver = webdriver.Chrome(executable_path=chromedriver)
elif (platform.system() == 'Linux'):
chromedriver = "./experiment/chromedriver/chromedriver_linux"
os.environ["webdriver.chrome.driver"] = chromedriver
chrome_option = webdriver.ChromeOptions()
chrome_option.add_argument("--proxy-server=yogi.pdl.cmu.edu:3128" )
self.driver = webdriver.Chrome(executable_path=chromedriver, chrome_options=chrome_option)
else:
print "Unidentified Platform"
sys.exit(0)
else:
print "Unsupported Browser"
sys.exit(0)
self.driver.implicitly_wait(10)
self.base_url = "https://www.google.com/"
self.verificationErrors = []
self.driver.set_page_load_timeout(40)
self.accept_next_alert = True
def test_webdriver(self):
fo = open(AD_FILE, "w")
fo.close()
driver = self.driver
driver.get(SITE)
count = 0
while(count < N):
els = driver.find_elements_by_css_selector("li.site-listing div.desc-container p.desc-paragraph a")
for el in els:
if(count < N):
t = el.get_attribute('innerHTML').lower()
fo = open(AD_FILE, "a")
fo.write(t + '\n')
fo.close()
count += 1
driver.find_element_by_css_selector("a.next").click()
def tearDown(self):
self.vdisplay.stop()
self.driver.quit()
class Webdriver(unittest.TestCase):
def setUp(self):
self.vdisplay = Xvfb(width=1280, height=720)
self.vdisplay.start()
# if(not vdisplay.start()):
# fo = open(LOG_FILE, "a")
# fo.write("Xvfbfailure||"+str(TREATMENTID)+"||"+str(ID)+"\n")
# fo.close()
# sys.exit(0)
if (BROWSER == 'firefox'):
if (platform.system() == 'Darwin'):
self.driver = webdriver.Firefox()
elif (platform.system() == 'Linux'):
self.driver = webdriver.Firefox(proxy=proxy)
else:
print "Unidentified Platform"
sys.exit(0)
elif (BROWSER == 'chrome'):
print "WARNING: Expecting chromedriver at specified location !!"
if (platform.system() == 'Darwin'):
chromedriver = "./experiment/chromedriver/chromedriver_mac"
os.environ["webdriver.chrome.driver"] = chromedriver
self.driver = webdriver.Chrome(executable_path=chromedriver)
elif (platform.system() == 'Linux'):
chromedriver = "./experiment/chromedriver/chromedriver_linux"
os.environ["webdriver.chrome.driver"] = chromedriver
chrome_option = webdriver.ChromeOptions()
chrome_option.add_argument(
"--proxy-server=yogi.pdl.cmu.edu:3128")
self.driver = webdriver.Chrome(executable_path=chromedriver,
chrome_options=chrome_option)
else:
print "Unidentified Platform"
sys.exit(0)
else:
print "Unsupported Browser"
sys.exit(0)
self.driver.implicitly_wait(10)
self.base_url = "https://www.google.com/"
self.verificationErrors = []
self.driver.set_page_load_timeout(40)
self.accept_next_alert = True
def test_webdriver(self):
driver = self.driver
helper.setLogFile(LOG_FILE)
helper.log("browserStarted||" + str(TREATMENTID), ID)
run = 0
while (run < RUNS):
helper.applyTreatment(driver, TREATMENTS[TREATMENTID], ID,
TREATMENTID)
helper.wait_for_others(AGENTS, ID, ROUND)
time.sleep(20)
helper.collectMeasurement(driver, MEASUREMENT, ID, TREATMENTID)
run = run + 1
def tearDown(self):
self.vdisplay.stop()
self.driver.quit()
class PBSeleniumTest(unittest.TestCase):
def setUp(self):
env = os.environ
self.browser_bin = env.get("BROWSER_BIN", "") # o/w use WD's default
if "TRAVIS" in os.environ:
self.xvfb = 1
else:
# by default don't use XVFB if we are not running on CI
self.xvfb = int(env.get("ENABLE_XVFB", 0))
self.pb_ext_path = self.get_extension_path() # path to the extension
if self.xvfb:
self.vdisplay = Xvfb(width=1280, height=720)
self.vdisplay.start()
self.driver = self.get_chrome_driver()
self.js = self.driver.execute_script
def load_url(self, url, wait_on_site=0):
"""Load a URL and wait before returning."""
self.driver.get(url)
sleep(wait_on_site)
def get_extension_path(self):
"""Return the path to the extension to be tested."""
if "PB_EXT_PATH" in os.environ:
return os.environ["PB_EXT_PATH"]
else: # check the default path if PB_EXT_PATH env. variable is empty
print "Can't find the env. variable PB_EXT_PATH, will check ../.."
# if the PB_EXT_PATH environment variable is not set
# check the default location for the last modified crx file
exts = glob("../../*.crx") # get matching files
return max(exts, key=os.path.getctime) if exts else ""
def txt_by_css(self, css_selector, timeout=SEL_DEFAULT_WAIT_TIMEOUT):
"""Find an element by CSS selector and return it's text."""
return self.find_el_by_css(css_selector, timeout).text
def find_el_by_css(self, css_selector, timeout=SEL_DEFAULT_WAIT_TIMEOUT):
return WebDriverWait(self.driver, timeout).until(
EC.presence_of_element_located((By.CSS_SELECTOR, css_selector)))
def get_chrome_driver(self):
"""Setup and return a Chrom[e|ium] browser for Selenium."""
opts = Options()
absp = os.path.abspath
if "TRAVIS" in os.environ: # github.com/travis-ci/travis-ci/issues/938
opts.add_argument("--no-sandbox")
opts.add_extension(self.pb_ext_path) # will fail if ext can't be found
if self.browser_bin: # otherwise will use webdriver's default binary
print "Browser binary:", absp(self.browser_bin)
opts.binary_location = self.browser_bin # set binary location
# Fix for https://code.google.com/p/chromedriver/issues/detail?id=799
opts.add_experimental_option("excludeSwitches",
["ignore-certificate-errors"])
return webdriver.Chrome(chrome_options=opts)
def tearDown(self):
self.driver.quit()
if self.xvfb and self.vdisplay:
self.vdisplay.stop()
def get_news(conn, max_date, current_time):
"""
华尔街见闻抓取
:param conn:
:param max_date: 数据库中最新新闻的日期
:param current_time: 当前时间
:return:
"""
func_name = "采集华尔街见闻"
logger.debug('start %s ' % func_name)
spider_data = datetime.datetime.strptime(current_time, '%Y-%m-%d %H:%M:%S')
driver = None
try:
xvfb = Xvfb(width=1280, height=720)
xvfb.start()
driver = webdriver.Firefox(executable_path=chromedriver_path)
driver.get('https://wallstreetcn.com/live/global')
# 让页面滚动到下面,window.scrollBy(0, scrollStep),ScrollStep ：间歇滚动间距
js = 'window.scrollBy(0,3000)'
driver.execute_script(js)
time.sleep(5)
js = 'window.scrollBy(0,60000)'
driver.execute_script(js)
time.sleep(5)
pages = driver.page_source
soup = BeautifulSoup(pages, 'html.parser')
soup1 = soup.find('div', class_='livenews-main')
content = soup1.find_all('div', class_='live-item')
news_source = '华尔街见闻'
news_type = '宏观'
last_news_time = '23:59'
d_date = datetime.datetime.strptime(current_time, '%Y-%m-%d %H:%M:%S')
for cont in content:
news_time = cont.find('time', attrs={'class': 'live-item_created'}).get_text()
news = cont.find('div', attrs={'class': 'live-item_main'}).find('div', attrs={'class': 'live-item_html'})
if news is None:
return
news = news.get_text().strip().replace('//', '')
if last_news_time < news_time:
d_date = d_date - datetime.timedelta(days=1)
s_date = d_date.strftime("%Y-%m-%d")
over_time = s_date + ' ' + news_time
if max_date > over_time:
break
sql_params = [over_time, spider_data, news_source, news_type, news]
logger.debug(sql_cj)
logger.debug(sql_params)
execute_sql(conn, sql_cj, sql_params)
last_news_time = news_time
logger.debug('end %s ' % func_name)
except Exception as e:
msg = func_name + ' 处理失败: ' + str(e)
logger.error(msg)
finally:
if driver:
# driver.close()
driver.quit()
xvfb.stop()
def test_with_xvfb():
if use_xvfb:
from xvfbwrapper import Xvfb
display = Xvfb(width=1920, height=1080)
display.start()
my_test()
if use_xvfb:
display.stop()
def test_with_xvfb():
if use_xvfb:
from xvfbwrapper import Xvfb
display = Xvfb(width=1920, height=1080)
display.start()
my_test()
if use_xvfb:
display.stop()
def test_stop(self):
orig_display = os.environ['DISPLAY']
xvfb = Xvfb()
xvfb.start()
self.assertNotEqual(orig_display, os.environ['DISPLAY'])
xvfb.stop()
self.assertIsNone(xvfb.proc)
self.assertEqual(orig_display, os.environ['DISPLAY'])
def test_stop(self):
orig_display = os.environ['DISPLAY']
xvfb = Xvfb()
xvfb.start()
self.assertNotEqual(orig_display, os.environ['DISPLAY'])
xvfb.stop()
self.assertEqual(orig_display, os.environ['DISPLAY'])
self.assertIsNone(xvfb.proc)
def xvfb(self, line, cell=None):
display = Xvfb(**self.xvfb_kwargs)
display.start()
if cell is None:
self.shell.ex(line)
else:
self.shell.ex(cell)
display.stop()
class Webdriver(unittest.TestCase):
def setUp(self):
self.vdisplay = Xvfb(width=1280, height=720)
self.vdisplay.start()
# if(not vdisplay.start()):
# fo = open(LOG_FILE, "a")
# fo.write("Xvfbfailure||"+str(TREATMENTID)+"||"+str(ID)+"\n")
# fo.close()
# sys.exit(0)
if(BROWSER=='firefox'):
if (platform.system()=='Darwin'):
self.driver = webdriver.Firefox()
elif (platform.system()=='Linux'):
self.driver = webdriver.Firefox(proxy=proxy)
else:
print "Unidentified Platform"
sys.exit(0)
elif(BROWSER=='chrome'):
print "WARNING: Expecting chromedriver at specified location !!"
if (platform.system()=='Darwin'):
chromedriver = "./experiment/chromedriver/chromedriver_mac"
os.environ["webdriver.chrome.driver"] = chromedriver
self.driver = webdriver.Chrome(executable_path=chromedriver)
elif (platform.system() == 'Linux'):
chromedriver = "./experiment/chromedriver/chromedriver_linux"
os.environ["webdriver.chrome.driver"] = chromedriver
chrome_option = webdriver.ChromeOptions()
chrome_option.add_argument("--proxy-server=yogi.pdl.cmu.edu:3128" )
self.driver = webdriver.Chrome(executable_path=chromedriver, chrome_options=chrome_option)
else:
print "Unidentified Platform"
sys.exit(0)
else:
print "Unsupported Browser"
sys.exit(0)
self.driver.implicitly_wait(10)
self.base_url = "https://www.google.com/"
self.verificationErrors = []
self.driver.set_page_load_timeout(40)
self.accept_next_alert = True
def test_webdriver(self):
driver = self.driver
cole.optIn(driver)
driver.get(SITE)
time.sleep(20)
#raw_input("wait")
pref = cole.get_ad_pref(driver)
print SITE
print 'pref=', pref
if pref != []:
fo = open(TARGET_FILE, "a")
fo.write(SITE+"||"+"@".join(pref)+'\n')
fo.close()
def tearDown(self):
self.vdisplay.stop()
self.driver.quit()
class MpvTestCase(unittest.TestCase):
def setUp(self):
self.disp = Xvfb()
self.disp.start()
self.m = mpv.MPV(vo='x11')
def tearDown(self):
self.m.terminate()
self.disp.stop()
def xvfb(request):
from xvfbwrapper import Xvfb
vdisplay = Xvfb()
vdisplay.start()
yield
vdisplay.stop()
class MpvTestCase(unittest.TestCase):
def setUp(self):
self.disp = Xvfb()
self.disp.start()
self.m = mpv.MPV(vo=testvo, loglevel='debug', log_handler=timed_print())
def tearDown(self):
self.m.terminate()
self.disp.stop()
def test_xvfb():
'''provide xvfb in test environment'''
vdisplay = Xvfb()
try: # guard for multiprocessing dist test
vdisplay.start()
yield vdisplay
vdisplay.stop()
except Exception as e:
yield vdisplay
class OFMTransferSiteManager(OFMSiteManager):
def __init__(self, user=None): # pylint: disable=super-init-not-called
self.user = user
if self.user:
self._login_user = self.user.ofm_username
self._login_password = self.user.ofm_password
else:
self._login_user = os.environ('OFM_USERNAME')
self._login_password = os.environ('OFM_PASSWORD')
self.display = Xvfb()
self.display.start()
def download_transfer_excels(self, matchdays=None):
profile = webdriver.FirefoxProfile(os.path.join(BASE_DIR, 'ofm_transfer_data', 'firefox_profile'))
profile.set_preference("browser.download.dir", os.path.join(BASE_DIR, 'ofm_transfer_data'))
self.browser = webdriver.Firefox(firefox_profile=profile)
self.browser.set_page_load_timeout(10)
self.login()
if not matchdays:
matchdays = [Matchday.get_current()]
for matchday in matchdays:
if not self._is_transfer_file_present(matchday):
try:
self._jump_to_transfer_page(self, matchday=matchday) # pylint: disable=redundant-keyword-arg
except TimeoutError:
pass
@staticmethod
def _is_transfer_file_present(matchday=None):
if not matchday:
matchday = Matchday.get_current()
if os.path.isfile(os.path.join(BASE_DIR,
'ofm_transfer_data',
'ofm_spielerwechsel_{}_{}.csv'.format(
matchday.season.number,
matchday.number)
)):
return True
return False
@timeout(5, use_signals=False)
def _jump_to_transfer_page(self, matchday=None):
if not matchday:
self.jump_to_frame(Constants.Transfer.DOWNLOAD_TRANSFERS)
else:
self.jump_to_frame(Constants.Transfer.DOWNLOAD_TRANSFERS_FROM_MATCHDAY.format(matchday.number))
def kill_browser(self):
if self.browser:
self.browser.stop_client()
self.display.stop()
def test_renderer(self):
vdisplay = Xvfb()
vdisplay.start()
ren = window.Renderer()
window.record(ren, n_frames=1, out_path=self.out_file, size=(600, 600))
self.assertTrue(os.path.exists(self.out_file))
vdisplay.stop()
def test_renderer(self):
vdisplay = Xvfb()
vdisplay.start()
ren = window.Renderer()
window.record(ren, n_frames=1, out_path=self.out_file, size=(600, 600))
self.assertTrue(os.path.exists(self.out_file))
vdisplay.stop()
def cli(debug, key, xvfb, verbose, first_name, last_name, email_address,
job_description, resume, job_location):
"""Apply to jobs automatically with Job Automate.
Job Automate requires the user's first name, last name, email address, job description,
and resume location prior to automating a job search. The job location is an optional argument
that is left blank by default. This allows Job Automate to search for jobs across the
entire United States. Each of the command line arguments requires a string data type. In
order to pass a string to each argument, type each argument inside quotation marks.
An example usage: jobautomate --key ABC123 "Homer" "Simpson" "*****@*****.**"
"Nuclear Technician" "/path/to/resume.txt" "Springfield"
When run, jobautomate will print to terminal the job position and company name tied to
the easily apply application. With the --verbose flag, jobautomate will also print to
terminal the applications which are not easily apply applications.
"""
if xvfb:
vdisplay = Xvfb()
vdisplay.start()
user_parameters = indeed_parameters(job_description, job_location)
response = access_indeed_api(user_parameters, key)
if response == {'error': 'Invalid publisher number provided.'}:
raise InvalidAPIKey(response)
driver = webdriver.Firefox()
while True:
for url in retrieve_indeed_urls(response):
driver.get(url)
try:
open_application(driver, 'indeed-apply-button')
switch_frames(driver, 'iframe[name$=modal-iframe]')
fill_application(driver, first_name, last_name, email_address,
resume)
if debug:
submit_application(driver, debug=True)
else:
submit_application(driver)
except (NoSuchElementException, ElementNotVisibleException):
if verbose:
print('Not an easily apply job application.')
else:
pass
user_prompt = click.prompt(
'Would you like to continue searching for jobs? (yes/no)')
if user_prompt == 'yes':
user_parameters['start'] += 25
response = access_indeed_api(user_parameters, key)
else:
driver.quit()
if xvfb:
vdisplay.stop()
break
class Webdriver(unittest.TestCase):
def setUp(self):
self.vdisplay = Xvfb(width=1280, height=720)
self.vdisplay.start()
# if(not vdisplay.start()):
# fo = open(LOG_FILE, "a")
# fo.write("Xvfbfailure||"+str(TREATMENTID)+"||"+str(ID)+"\n")
# fo.close()
# sys.exit(0)
if(BROWSER=='firefox'):
if (platform.system()=='Darwin'):
self.driver = webdriver.Firefox()
elif (platform.system()=='Linux'):
self.driver = webdriver.Firefox(proxy=proxy)
else:
print "Unidentified Platform"
sys.exit(0)
elif(BROWSER=='chrome'):
print "WARNING: Expecting chromedriver at specified location !!"
if (platform.system()=='Darwin'):
chromedriver = "./experiment/chromedriver/chromedriver_mac"
os.environ["webdriver.chrome.driver"] = chromedriver
self.driver = webdriver.Chrome(executable_path=chromedriver)
elif (platform.system() == 'Linux'):
chromedriver = "./experiment/chromedriver/chromedriver_linux"
os.environ["webdriver.chrome.driver"] = chromedriver
chrome_option = webdriver.ChromeOptions()
chrome_option.add_argument("--proxy-server=yogi.pdl.cmu.edu:3128" )
self.driver = webdriver.Chrome(executable_path=chromedriver, chrome_options=chrome_option)
else:
print "Unidentified Platform"
sys.exit(0)
else:
print "Unsupported Browser"
sys.exit(0)
self.driver.implicitly_wait(10)
self.base_url = "https://www.google.com/"
self.verificationErrors = []
self.driver.set_page_load_timeout(40)
self.accept_next_alert = True
def test_webdriver(self):
driver = self.driver
helper.setLogFile(LOG_FILE)
helper.log("browserStarted||"+str(TREATMENTID), ID)
run = 0
while (run < RUNS):
helper.applyTreatment(driver, TREATMENTS[TREATMENTID], ID, TREATMENTID)
helper.wait_for_others(AGENTS, ID, ROUND)
time.sleep(20)
helper.collectMeasurement(driver, MEASUREMENT, ID, TREATMENTID)
run = run+1
def tearDown(self):
self.vdisplay.stop()
self.driver.quit()
def suro_ws_js_enabled_page():
vdisplay = Xvfb()
vdisplay.start()
browser = webdriver.Firefox()
browser.get('http://www.google.com')
print browser.title
browser.quit()
vdisplay.stop()
def suro_ws_js_enabled_page():
vdisplay = Xvfb()
vdisplay.start()
browser = webdriver.Firefox()
browser.get('http://www.google.com')
print browser.title
browser.quit()
vdisplay.stop()
def html_to_png(filepaths, width=1280, height=720,save=False):
"""
Takes in input one or a list of paths to html files containing a cytoscape network under variable cy. Returns the base64 of the png of the network.
N.B. Common possible error :
- (variable cy not found) can be caused by unvalidity of the source of the cytoscape.js in the html file.
:param filepaths: can be one or a list of path to html network files. absolute and relative path are supported.
:param width: the image width
:param height: the image height
:param save: if True, the generated png will be saved in the same directory then its html counterpart.
:return: a list of png encoded as base 64 unicode strings
"""
from selenium import webdriver
from xvfbwrapper import Xvfb
import os
is_list = hasattr(filepaths, "__iter__")
if not is_list:
assert type(filepaths) is str
filepaths = [filepaths]
#Xvfb for headless mode
xvfb = Xvfb(width=width, height=height)
xvfb.start()
browser = webdriver.Firefox()
def cast_html_to_png(file_location):
browser.get("file://"+file_location)
time.sleep(1)
b64 = browser.execute_script('return cy.png()')
#index is always 22
index = b64.index("base64,") + len("base64,")
return b64[index:]
abspath = os.getcwd()
png_images = []
for f in filepaths:
if f[0] != '/':
f = os.path.join(abspath,f)
# print("File : "+f)
b64image = cast_html_to_png(f)
png_images.append(b64image)
if save :
if len(f) > 5 and f[-5:] == ".html":
f = f[:-5]
f += ".png"
save_png(b64image,f)
browser.quit()
xvfb.stop()
return png_images
class BaseLiveTest(LiveServerTestCase):
@classmethod
def setUp(self):
if RUN_TESTS_IN_BACKGROUND:
self.xvfb = Xvfb(width=1280, height=720)
self.xvfb.start()
self.browser = webdriver.Chrome()
@classmethod
def tearDown(self):
self.browser.quit()
if RUN_TESTS_IN_BACKGROUND:
self.xvfb.stop()
# This function performs an implicit test that the user can login. This is used in the base
# test class, as the login operation is required in the majority of the test cases.
def can_login(self, email='', password='', login_first_time=True):
if login_first_time:
# User opens the web browser and goes to RGT home page
self.browser.get(self.live_server_url + '/home/')
# User sees the RGT login page
body = self.browser.find_element_by_tag_name('body')
self.assertIn('Login', body.text)
# User types email and password and hits enter
email_field = self.browser.find_element_by_name('email')
email_field.send_keys(email)
password_field = self.browser.find_element_by_name('password')
password_field.send_keys(password)
password_field.send_keys(Keys.RETURN)
# email and password are accepted, and the user is redirected to
# the Home page
body = self.browser.find_element_by_tag_name('body')
self.assertIn('Welcome', body.text)
def wait_for_dialog_box_with_message(self, message=''):
# wait for the dialog box to appear with the desired message
WebDriverWait(
self.browser,
10).until(lambda x: self.browser.find_element_by_css_selector(
"div[class*='ui-dialog'][style*='block']"))
dialog_box = self.browser.find_element_by_class_name("ui-dialog")
self.assertIn(message, dialog_box.text)
# User clicks the close button to close the dialog
close_dialog_button = self.browser.find_element_by_css_selector(
"button[role='button']")
close_dialog_button.click()
# Do nothing after the dialog box disappeared
time.sleep(1)
def getHtml2(url):
try:
vdisplay = Xvfb()
vdisplay.start()
browser = webdriver.Firefox()
browser.set_page_load_timeout(120)
browser.get(url)
return browser.page_source
except Exception as e:
print(e)
finally:
vdisplay.stop()
class BaseLiveTest(LiveServerTestCase):
@classmethod
def setUp(self):
if RUN_TESTS_IN_BACKGROUND:
self.xvfb = Xvfb(width=1280, height=720)
self.xvfb.start()
self.browser = webdriver.Chrome()
@classmethod
def tearDown(self):
self.browser.quit()
if RUN_TESTS_IN_BACKGROUND:
self.xvfb.stop()
# This function performs an implicit test that the user can login. This is used in the base
# test class, as the login operation is required in the majority of the test cases.
def can_login(self, email='', password='', login_first_time=True):
if login_first_time:
# User opens the web browser and goes to RGT home page
self.browser.get(self.live_server_url + '/home/')
# User sees the RGT login page
body = self.browser.find_element_by_tag_name('body')
self.assertIn('Login', body.text)
# User types email and password and hits enter
email_field = self.browser.find_element_by_name('email')
email_field.send_keys(email)
password_field = self.browser.find_element_by_name('password')
password_field.send_keys(password)
password_field.send_keys(Keys.RETURN)
# email and password are accepted, and the user is redirected to
# the Home page
body = self.browser.find_element_by_tag_name('body')
self.assertIn('Welcome', body.text)
def wait_for_dialog_box_with_message(self, message=''):
# wait for the dialog box to appear with the desired message
WebDriverWait(self.browser, 10).until(
lambda x: self.browser.find_element_by_css_selector("div[class*='ui-dialog'][style*='block']"))
dialog_box = self.browser.find_element_by_class_name("ui-dialog")
self.assertIn(message, dialog_box.text)
# User clicks the close button to close the dialog
close_dialog_button = self.browser.find_element_by_css_selector("button[role='button']")
close_dialog_button.click()
# Do nothing after the dialog box disappeared
time.sleep(1)
def screengrab_firefox(url):
"""get screenshot: firefox with adblock
and virtual display"""
vdisplay = Xvfb()
vdisplay.start()
fp = webdriver.FirefoxProfile()
fp.add_extension('adblockplusfirefox.xpi')
fp.set_preference("extensions.adblockplus.currentVersion", "2.4")
fox = webdriver.Firefox(fp)
fox.get(url)
fox.save_screenshot('ToIBot.png')
fox.quit()
vdisplay.stop()
class PBSeleniumTest(unittest.TestCase):
def setUp(self):
env = os.environ
self.browser_bin = env.get("BROWSER_BIN", "") # o/w use WD's default
self.xvfb = int(env.get("ENABLE_XVFB", 1)) # enabled by default
self.pb_ext_path = self.get_extension_path() # path to the extension
if self.xvfb:
self.vdisplay = Xvfb(width=1280, height=720)
self.vdisplay.start()
self.driver = self.get_chrome_driver()
self.driver.implicitly_wait(10)
self.js = self.driver.execute_script
def get_extension_path(self):
"""Return the path to the extension to be tested."""
if "PB_EXT_PATH" in os.environ:
return os.environ["PB_EXT_PATH"]
else: # check the default path if PB_EXT_PATH env. variable is empty
print "Can't find the environment variable PB_EXT_PATH"
# if the PB_EXT_PATH environment variable is not set
# check the default location for the last modified crx file
exts = glob("../../*.crx") # get matching files
return max(exts, key=os.path.getctime) if exts else ""
def txt_by_css(self, css_selector):
"""Find an element by CSS selector and return it's text."""
return self.driver.find_element_by_css_selector(css_selector).text
def get_chrome_driver(self):
"""Setup and return a Chrom[e|ium] browser for Selenium."""
opts = Options()
absp = os.path.abspath
if "TRAVIS" in os.environ: # github.com/travis-ci/travis-ci/issues/938
opts.add_argument("--no-sandbox")
opts.add_extension(self.pb_ext_path) # will fail if ext can't be found
if self.browser_bin: # otherwise will use webdriver's default binary
print "Browser binary:", absp(self.browser_bin)
opts.binary_location = self.browser_bin # set binary location
# Fix for https://code.google.com/p/chromedriver/issues/detail?id=799
opts.add_experimental_option("excludeSwitches",
["ignore-certificate-errors"])
return webdriver.Chrome(chrome_options=opts)
def tearDown(self):
self.driver.quit()
if self.xvfb and self.vdisplay:
self.vdisplay.stop()
class Headless(Browser):
def __init__(self):
from xvfbwrapper import Xvfb
self.width = runtime.settings.SELENIUM_ARGS.get('WINDOW_SIZE_WIDTH')
if self.width == False:
self.width = 1024
self.height = runtime.settings.SELENIUM_ARGS.get('WINDOW_SIZE_HEIGHT')
if self.height == False:
self.height = 768
self._vdisplay = Xvfb(width=self.width,height=self.height)
self._vdisplay.start()
super(Headless, self).__init__()
super(Headless, self).__init__()
def clear(self):
super(Headless, self).clear()
self._vdisplay.stop()
class Surfer:
"""This class does virtual web surfing on our demand
"""
def __init__(self,delay=2):
self.extraDelay = delay #extra time to wait after each operation (s)
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait # available since 2.4.0
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as ExpectedConditions
from xvfbwrapper import Xvfb
self.vdisplay = Xvfb()
self.vdisplay.start()
self.selenium_profile = webdriver.FirefoxProfile()
self.selenium_driver = webdriver.Firefox(self.selenium_profile)
def surfTo(self,url):
self.selenium_driver.get(url)
self.selenium_driver.implicitly_wait(self.extraDelay)
def clickOnStuff(self,xPath):
elementList = self.selenium_driver.find_elements_by_xpath(xPath)
if not elementList:
logging.warning("No elements found for xPath `%s`" % xPath)
else:
for element in elementList:
element.click()
self.selenium_driver.implicitly_wait(self.extraDelay)
def findElements(self,xPath):
return self.selenium_driver.find_elements_by_xpath(xPath)
def getUrlList(self,xPath):
urlList = []
elementList = self.findElements(xPath)
for element in elementList:
href = element.get_attribute("href")
if href is not None:
urlList.append(Url(href))
return urlList#list of Url objects
def kill(self):
self.selenium_driver.close()
self.vdisplay.stop()
class TestHomepages(unittest.TestCase):
def setUp(self):
self.vdisplay = Xvfb(width=800, height=600)
self.vdisplay.start()
self.browser = webdriver.Firefox()
def testUbuntuHomepage(self):
self.browser.get('http://www.ubuntu.com')
self.assertIn('Ubuntu', self.browser.title)
def testGoogleHomepage(self):
self.browser.get('http://www.google.com')
self.assertIn('Google', self.browser.title)
def tearDown(self):
self.browser.quit()
self.vdisplay.stop()
def getAuth():
import requests
import settings
from time import sleep
from splinter import Browser
from xvfbwrapper import Xvfb
vdisplay = Xvfb(width=3840, height=2160)
vdisplay.start()
client_id = settings.getClientID()
email = settings.getEmail()
password = settings.getPassword()
team_name = settings.getTeamName()
scopes = ('users:read', 'channels:write', 'channels:history', 'channels:read',
'chat:write:bot', 'chat:write:bot', 'chat:write:bot', 'dnd:write',
'dnd:read', 'emoji:read', 'files:write:user', 'files:read', 'groups:write',
'groups:history', 'groups:read', 'im:write', 'im:history', 'im:read',
'mpim:write', 'mpim:history', 'mpim:read', 'pins:write', 'pins:read',
'reactions:write', 'reactions:read', 'reminders:write', 'reminders:read',
'search:read', 'stars:write', 'stars:read', 'team:read', 'usergroups:write',
'usergroups:read', 'users:write', 'admin')
scopeStr = '+'.join(scopes)
browser = Browser('chrome')
url = 'https://slack.com/oauth/authorize?client_id={}&scope={}&state=test'.format(client_id, scopeStr)
browser.visit(url)
browser.fill('domain', team_name)
browser.find_by_id('submit_team_domain').first.click()
browser.fill('email', email)
browser.fill('password', password)
browser.find_by_id('signin_btn').first.click()
button = browser.find_by_id('oauth_authorizify')
button.mouse_over()
button.click()
codeStart = browser.url.find('=') + 1
codeEnd = browser.url.find('&')
code = browser.url[codeStart:codeEnd]
browser.quit()
vdisplay.stop()
return code
def cli(debug, key, xvfb, verbose, first_name, last_name, email_address, job_description, resume, job_location):
"""Apply to jobs automatically with Job Automate.
Job Automate requires the user's first name, last name, email address, job description,
and resume location prior to automating a job search. The job location is an optional argument
that is left blank by default. This allows Job Automate to search for jobs across the
entire United States. Each of the command line arguments requires a string data type. In
order to pass a string to each argument, type each argument inside quotation marks.
An example usage: jobautomate "Homer" "Simpson" "*****@*****.**" "Nuclear Technician"
"/path/to/resume.txt" "Springfield"
When run, jobautomate will print to terminal the job position and company name tied to
the easily apply application. With the --verbose flag, jobautomate will also print to
terminal the applications which are not easily apply applications.
"""
if xvfb:
vdisplay = Xvfb()
vdisplay.start()
driver = webdriver.Firefox()
user_parameters = indeed_parameters(job_description, job_location)
while True:
for url in indeed_urls(user_parameters, key):
driver.get(url)
try:
find_apply_button(driver, "indeed-apply-button")
switch_frames(driver, "iframe[name$=modal-iframe]")
fill_application(driver, first_name, last_name, email_address, resume)
if debug:
apply_or_continue(driver, debug=True)
except (NoSuchElementException, ElementNotVisibleException):
if verbose:
print("Not an easily apply job application.")
else:
pass
user_prompt = click.prompt("Would you like to continue searching for jobs? (yes/no")
if user_prompt == "yes":
user_parameters["start"] += 25
else:
driver.quit()
if xvfb:
vdisplay.stop()
break
def return_html_code(url, proxy_use):
vdisplay = Xvfb()
vdisplay.start()
proxy_address_list = get_proxy_fastest()
if proxy_address_list != False:
proxy_address = random.choice(proxy_address_list)
ip, port = proxy_address.split(":")
print ip, port
profile = webdriver.FirefoxProfile()
profile.set_preference("network.proxy.http", ip)
profile.set_preference("network.proxy.http_port", port)
profile.set_preference("network.proxy_type", 1)
driver = webdriver.Firefox(firefox_profile=profile)
else:
print "Using localhost, unable to get proxy"
driver = webdriver.Firefox()
driver.maximize_window()
driver.get(url)
# initial wait for the tweets to load
# initial wait for the tweets to load
wait = WebDriverWait(driver, 30)
try:
wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "li[data-item-id]")))
except TimeoutException:
driver.quit()
return False # scroll down to the last tweet until there is no more tweets loaded
while True:
tweets = driver.find_elements_by_css_selector("li[data-item-id]")
print len(tweets) # added in edit 1
number_of_tweets = len(tweets)
driver.execute_script("arguments[0].scrollIntoView(true);", tweets[-1])
try:
wait.until(
wait_for_more_than_n_elements_to_be_present((By.CSS_SELECTOR, "li[data-item-id]"), number_of_tweets)
)
except TimeoutException:
break
html_full_source = driver.page_source
driver.close()
vdisplay.stop()
return html_full_source
def get_dynamic_html(url):
"""
Function to get dynamic html of given url
"""
if not dynamic_dependencies:
print "The extra depencies to scrape aren't"\
" installed\n Please make sure you install them\n"\
"(selenium , xvfbwrapper for python are the "\
"extra dependencies)"
else:
source = ''
virtual_display = Xvfb()
virtual_display.start() # Start virtual display
browser = webdriver.Firefox()
print "Finding the given page...."
browser.get(url)
source = browser.page_source
virtual_display.stop()
return source
class Screenshot(object):
def __init__(self, output, size=(1200, 600), wait=1000):
self.output = output
self.size = size
self.wait = wait
def __enter__(self):
self.vdisplay = Xvfb(width=self.size[0], height=self.size[1])
self.vdisplay.start()
self.app = wx.App(False)
wx.UIActionSimulator().MouseMove(0, 0)
return self
def set_window(self, window):
self.window = window
def __exit__(self, type, value, traceback):
def _screenshot():
rect = self.window.GetRect()
if sys.platform == 'linux2':
client_x, client_y = self.window.ClientToScreen((0, 0))
border_width = client_x - rect.x
title_bar_height = client_y - rect.y
rect.width += (border_width * 2)
rect.height += title_bar_height + border_width
dcScreen = wx.ScreenDC()
bmp = wx.EmptyBitmap(rect.width, rect.height)
memDC = wx.MemoryDC()
memDC.SelectObject(bmp)
memDC.Blit(0, 0, rect.width, rect.height, dcScreen, rect.x, rect.y)
memDC.SelectObject(wx.NullBitmap)
img = bmp.ConvertToImage()
img.SaveFile(self.output, wx.BITMAP_TYPE_PNG)
wx.FutureCall(self.wait, _screenshot)
wx.FutureCall(self.wait + 1000, self.window.Destroy)
self.app.MainLoop()
self.vdisplay.stop()
def startUser(name, url, username, password, headless, logDir, chromedriver,
workHour):
if headless:
xvfb = Xvfb(width=1100, height=800)
xvfb.start()
user = User(name, url=url, username=username, password=password,
logDir=logDir, chromedriver=chromedriver, workHour=workHour)
# TODO - configure workflow
# Always start with Login() and end with Logout(). There has to be at least
# one workflow between Login() and Logout().
user.addWorkflow([
LogInOutWorkflow.Login(),
MonitorDashboard(),
MonitorEvents(),
MonitorDashboard(),
MonitorEvents(),
MonitorDashboard(),
MonitorEvents(),
MonitorDashboard(),
MonitorEvents(),
LogInOutWorkflow.Logout()])
try:
user.work()
except:
user.log("%s unexpectedly failed" % user.name)
traceback.print_exc()
finally:
#log results
resultsStr = ""
for result in user.results:
resultsStr += str(result)
resultsStr += ","
user.log(resultsStr)
print "cleaning up %s" % user.name
user.quit()
if headless:
xvfb.stop()
class FrontendTestCase(LiveServerTestCase):
"""
Test frontend access through firefox webdriver
"""
def create_app(self):
return app
def setUp(self):
self.xvfb = Xvfb(width=1024, height=768)
self.xvfb.start()
self.browser = webdriver.Firefox()
def tearDown(self):
self.browser.quit()
self.xvfb.stop()
def test_index_access(self):
"""
Try to Access the first page.
:return:
"""
url = self.get_server_url()
self.browser.get(url)
# Can we see the right value on title bar?
self.assertIn('Home', self.browser.title)
# Can we find a specific page?
self.browser.find_element_by_link_text('Find Page').click()
self.assertIn('find-page', self.browser.current_url)
self.fail('Implement Find Page.')
self.browser.find_element_by_id('search-field').text('index')
self.browser.find_element_by_link_text('Search').click()
self.browser.find_elements_by_link_text('Index')
