class Test(TestCase):
def wait(self):
self.screen.waitgrab()
def setUp(self):
self.screen = SmartDisplay()
self.screen.start()
self.p = None
def tearDown(self):
self.p.stop()
self.screen.stop()
# def test_empty(self):
# self.p = EasyProcess('zenity --warning').start()
# wnd is not ready
# self.assertRaises(EmptyScreenException, tab_rectangles)
# def test_zenity(self):
# self.p = EasyProcess('zenity --warning').start()
# self.wait()
# ls = tab_rectangles()
# self.assertEquals(len(ls), 2)
def test_notab(self):
self.p = EasyProcess('xmessage hi').start()
self.wait()
ls = tab_rectangles()
self.assertEquals(len(ls), 0)
def test_gmessage(self):
self.p = EasyProcess('gmessage -buttons x,y,z hi').start()
self.wait()
ls = tab_rectangles()
self.assertEquals(len(ls), 4)
class Test(TestCase):
def wait(self):
self.screen.waitgrab()
def setUp(self):
self.screen = SmartDisplay()
self.screen.start()
def tearDown(self):
self.p.stop()
self.screen.stop()
# def test_empty(self):
# self.p = EasyProcess('zenity --warning').start()
# wnd is not ready
# time.sleep(0.2)
# self.assertRaises(EmptyScreenException, active_rectangles)
def test_zenity(self):
self.p = EasyProcess('zenity --warning').start()
self.wait()
ls = active_rectangles()
self.assertEquals(len(ls), 1)
def test_notab(self):
self.p = EasyProcess('xmessage -buttons x,y,z hi').start()
self.wait()
ls = active_rectangles(grid=10)
self.assertEquals(len(ls), 3)
def test_gmessage(self):
self.p = EasyProcess('gmessage -buttons x,y,z hi').start()
self.wait()
ls = active_rectangles()
self.assertEquals(len(ls), 3)
class VirtualDisplay(Service, Singletone):
"""
Selenium executor
:type virtual_display: Display
"""
SHARED_VIRTUAL_DISPLAY = None
def __init__(self):
super(VirtualDisplay, self).__init__()
self.virtual_display = None
def get_virtual_display(self):
return self.virtual_display
def set_virtual_display(self):
if is_windows():
self.log.warning("Cannot have virtual display on Windows, ignoring")
return
if VirtualDisplay.SHARED_VIRTUAL_DISPLAY:
self.virtual_display = VirtualDisplay.SHARED_VIRTUAL_DISPLAY
else:
width = self.parameters.get("width", 1024)
height = self.parameters.get("height", 768)
self.virtual_display = Display(size=(width, height))
msg = "Starting virtual display[%s]: %s"
self.log.info(msg, self.virtual_display.size, self.virtual_display.new_display_var)
self.virtual_display.start()
VirtualDisplay.SHARED_VIRTUAL_DISPLAY = self.virtual_display
self.engine.shared_env.set({'DISPLAY': os.environ['DISPLAY']}) # backward compatibility
def free_virtual_display(self):
if self.virtual_display and self.virtual_display.is_alive():
self.virtual_display.stop()
if VirtualDisplay.SHARED_VIRTUAL_DISPLAY:
VirtualDisplay.SHARED_VIRTUAL_DISPLAY = None
def startup(self):
self.set_virtual_display()
def check_virtual_display(self):
if self.virtual_display:
if not self.virtual_display.is_alive():
self.log.info("Virtual display out: %s", self.virtual_display.stdout)
self.log.warning("Virtual display err: %s", self.virtual_display.stderr)
raise TaurusInternalException("Virtual display failed: %s" % self.virtual_display.return_code)
def check(self):
self.check_virtual_display()
return False
def shutdown(self):
self.free_virtual_display()
class CuiJsTestCase(LiveServerTestCase):
def setUp(self):
# See https://code.djangoproject.com/ticket/10827
from django.contrib.contenttypes.models import ContentType
ContentType.objects.clear_cache()
if not os.environ.get('SHOW_SELENIUM'):
self.display = SmartDisplay(visible=0, size=(1024, 768))
self.display.start()
remote_selenium = os.environ.get('REMOTE_SELENIUM')
if not remote_selenium:
self.driver = webdriver.Firefox()
else:
self.driver = webdriver.Remote(
command_executor=remote_selenium,
desired_capabilities={
'browserName': 'unknown',
'javascriptEnabled': True,
'platform': 'ANY',
'version': ''
}
)
def tearDown(self):
if hasattr(self, 'driver'):
self.driver.quit()
if hasattr(self, 'display'):
self.display.stop()
def wait_until_expr(self, expr, timeout=60):
WebDriverWait(self.driver, timeout).until(
lambda driver: driver.execute_script('return (%s)' % expr))
def test(self):
sys.stderr.write('\n\nRunning candidate UI unit tests...\n')
sys.stderr.flush()
tests_url = self.live_server_url + reverse('cui_test')
jasmine_spec = os.environ.get('JASMINE_SPEC')
if jasmine_spec:
tests_url += "?spec={}".format(urlquote(jasmine_spec))
self.driver.get(tests_url)
self.wait_until_expr('window.seleniumReporter')
runner = SeleniumRunner(self.driver, sys.stderr)
success = runner.run()
self.assertTrue(success, 'JS tests failed. See full report on stderr')
class CuiJsTestCase(StaticLiveServerTestCase):
def setUp(self):
# See https://code.djangoproject.com/ticket/10827
from django.contrib.contenttypes.models import ContentType
ContentType.objects.clear_cache()
if not os.environ.get('SHOW_SELENIUM'):
self.display = SmartDisplay(visible=0, size=(1024, 768))
self.display.start()
remote_selenium = os.environ.get('REMOTE_SELENIUM')
if not remote_selenium:
self.driver = webdriver.Firefox()
else:
self.driver = webdriver.Remote(command_executor=remote_selenium,
desired_capabilities={
'browserName': 'unknown',
'javascriptEnabled': True,
'platform': 'ANY',
'version': ''
})
def tearDown(self):
if hasattr(self, 'driver'):
self.driver.quit()
if hasattr(self, 'display'):
self.display.stop()
def wait_until_expr(self, expr, timeout=60):
WebDriverWait(self.driver, timeout).until(
lambda driver: driver.execute_script('return (%s)' % expr))
def test(self):
sys.stderr.write('\n\nRunning candidate UI unit tests...\n')
sys.stderr.flush()
tests_url = self.live_server_url + reverse('cui_test')
jasmine_spec = os.environ.get('JASMINE_SPEC')
if jasmine_spec:
tests_url += "?spec={}".format(urlquote(jasmine_spec))
self.driver.get(tests_url)
self.wait_until_expr('window.seleniumReporter')
runner = SeleniumRunner(self.driver, sys.stderr)
success = runner.run()
self.assertTrue(success, 'JS tests failed. See full report on stderr')
class Test(TestCase):
def wait(self):
self.screen.waitgrab()
def setUp(self):
self.screen = SmartDisplay(visible=VISIBLE)
self.screen.start()
self.p = None
def tearDown(self):
self.p.stop()
self.screen.stop()
def test_zenity(self):
self.p = EasyProcess('zenity --warning').start()
self.wait()
send_key('\n')
self.assertFalse(getbbox(grab()))
self.p = EasyProcess('zenity --warning').start()
self.wait()
send_key_list(['\n'])
self.assertFalse(getbbox(grab()))
self.p = EasyProcess('zenity --warning').start()
self.wait()
send_key(' ')
self.assertFalse(getbbox(grab()))
self.p = EasyProcess('zenity --warning').start()
self.wait()
send_key('x')
self.assertTrue(getbbox(grab()))
def test_gcalctool1(self):
self.p = EasyProcess('gnome-calculator').start()
self.wait()
focus_wnd()
send_key('ctrl+q')
time.sleep(1)
# img_debug(grab(), 'ctrl+q')
self.assertFalse(getbbox(grab()))
class RealBrowser(webdriver.Firefox):
def __init__(self, timeout=30):
self._abstract_display = SmartDisplay(visible=0)
self._abstract_display.start()
super(RealBrowser, self).__init__()
self.implicitly_wait(timeout)
def quit(self):
super(RealBrowser, self).quit()
self._abstract_display.stop()
@timeout(REQUEST_TIMEOUT)
def _open(self, url):
self.get(url)
def open(self, url):
try:
self._open(url)
return True
except Exception, e:
logger.error('network error for %s: %s', url, str(e))
class CuiJsTestCase(LiveServerTestCase):
def setUp(self):
if not os.environ.get('SHOW_SELENIUM'):
self.display = SmartDisplay(visible=0, size=(1024, 768))
self.display.start()
self.driver = webdriver.Firefox()
def tearDown(self):
if hasattr(self, 'driver'):
self.driver.quit()
if hasattr(self, 'display'):
self.display.stop()
def wait_until_expr(self, expr, timeout=60):
WebDriverWait(self.driver, timeout).until(
lambda driver: driver.execute_script('return (%s)' % expr))
def test(self):
sys.stderr.write('\n\nRunning candidate UI unit tests...\n')
sys.stderr.flush()
self.driver.get(self.live_server_url+reverse('cui_test'))
self.wait_until_expr('window.jsApiReporter !== undefined && window.jsApiReporter.finished')
specs = self.driver.execute_script('return window.jsApiReporter.specs()')
self.assertTrue(len(specs) > 0, 'No test results found! The tests probably contain syntax errors.')
passed = True
for spec in specs:
sys.stderr.write(' %s ... %s\n' % (spec['fullName'], spec['status']))
if spec['status'] != 'passed':
passed = False
for exp in spec['failedExpectations']:
sys.stderr.write(' %s\n' % exp['message'])
sys.stderr.write('Access full report at %s\n\n' % reverse('cui_test'))
sys.stderr.flush()
self.assertTrue(passed, 'JS tests failed. See full report on stderr')
class VirtualDisplay(Service, Singletone):
"""
Selenium executor
:type virtual_display: Display
"""
SHARED_VIRTUAL_DISPLAY = None
def __init__(self):
super(VirtualDisplay, self).__init__()
self.virtual_display = None
def get_virtual_display(self):
return self.virtual_display
def set_virtual_display(self):
if is_windows():
self.log.warning(
"Cannot have virtual display on Windows, ignoring")
return
if VirtualDisplay.SHARED_VIRTUAL_DISPLAY:
self.virtual_display = VirtualDisplay.SHARED_VIRTUAL_DISPLAY
else:
width = self.parameters.get("width", 1024)
height = self.parameters.get("height", 768)
self.virtual_display = Display(size=(width, height))
self.virtual_display.start()
msg = "Starting virtual display[%s] %s"
self.log.info(msg, (width, height),
self.virtual_display.new_display_var)
VirtualDisplay.SHARED_VIRTUAL_DISPLAY = self.virtual_display
self.engine.shared_env.set({'DISPLAY': os.environ['DISPLAY']
}) # backward compatibility
def free_virtual_display(self):
if self.virtual_display and self.virtual_display.is_alive():
self.virtual_display.stop()
if VirtualDisplay.SHARED_VIRTUAL_DISPLAY:
VirtualDisplay.SHARED_VIRTUAL_DISPLAY = None
def startup(self):
self.set_virtual_display()
def check_virtual_display(self):
if self.virtual_display:
if not self.virtual_display.is_alive():
self.log.info("Virtual display out: %s",
self.virtual_display.stdout)
self.log.warning("Virtual display err: %s",
self.virtual_display.stderr)
raise TaurusInternalException("Virtual display failed: %s" %
self.virtual_display.return_code)
def check(self):
self.check_virtual_display()
return False
def shutdown(self):
self.free_virtual_display()
class SumoEnvironment(gym.Env):
"""
SUMO Environment for Traffic Signal Control
:param net_file: (str) SUMO .net.xml file
:param route_file: (str) SUMO .rou.xml file
:param out_csv_name: (Optional[str]) name of the .csv output with simulation results. If None no output is generated
:param use_gui: (bool) Wheter to run SUMO simulation with GUI visualisation
:param virtual_display: (Optional[Tuple[int,int]]) Resolution of a virtual display for rendering
:param begin_time: (int) The time step (in seconds) the simulation starts
:param num_seconds: (int) Number of simulated seconds on SUMO. The time in seconds the simulation must end.
:param max_depart_delay: (int) Vehicles are discarded if they could not be inserted after max_depart_delay seconds
:param delta_time: (int) Simulation seconds between actions
:param min_green: (int) Minimum green time in a phase
:param max_green: (int) Max green time in a phase
:single_agent: (bool) If true, it behaves like a regular gym.Env. Else, it behaves like a MultiagentEnv (https://github.com/ray-project/ray/blob/master/python/ray/rllib/env/multi_agent_env.py)
:sumo_seed: (int/string) Random seed for sumo. If 'random' it uses a randomly chosen seed.
:fixed_ts: (bool) If true, it will follow the phase configuration in the route_file and ignore the actions.
:sumo_warnings: (bool) If False, remove SUMO warnings in the terminal
"""
CONNECTION_LABEL = 0 # For traci multi-client support
def __init__(
self,
net_file: str,
route_file: str,
out_csv_name: Optional[str] = None,
use_gui: bool = False,
virtual_display: Optional[Tuple[int,int]] = None,
begin_time: int = 0,
num_seconds: int = 20000,
max_depart_delay: int = 100000,
time_to_teleport: int = -1,
delta_time: int = 5,
yellow_time: int = 2,
min_green: int = 5,
max_green: int = 50,
single_agent: bool = False,
sumo_seed: Union[str,int] = 'random',
fixed_ts: bool = False,
sumo_warnings: bool = True,
):
self._net = net_file
self._route = route_file
self.use_gui = use_gui
if self.use_gui:
self._sumo_binary = sumolib.checkBinary('sumo-gui')
else:
self._sumo_binary = sumolib.checkBinary('sumo')
self.virtual_display = virtual_display
assert delta_time > yellow_time, "Time between actions must be at least greater than yellow time."
self.begin_time = begin_time
self.sim_max_time = num_seconds
self.delta_time = delta_time # seconds on sumo at each step
self.max_depart_delay = max_depart_delay # Max wait time to insert a vehicle
self.time_to_teleport = time_to_teleport
self.min_green = min_green
self.max_green = max_green
self.yellow_time = yellow_time
self.single_agent = single_agent
self.sumo_seed = sumo_seed
self.fixed_ts = fixed_ts
self.sumo_warnings = sumo_warnings
self.label = str(SumoEnvironment.CONNECTION_LABEL)
SumoEnvironment.CONNECTION_LABEL += 1
self.sumo = None
if LIBSUMO:
traci.start([sumolib.checkBinary('sumo'), '-n', self._net]) # Start only to retrieve traffic light information
conn = traci
else:
traci.start([sumolib.checkBinary('sumo'), '-n', self._net], label='init_connection'+self.label)
conn = traci.getConnection('init_connection'+self.label)
self.ts_ids = list(conn.trafficlight.getIDList())
self.traffic_signals = {ts: TrafficSignal(self,
ts,
self.delta_time,
self.yellow_time,
self.min_green,
self.max_green,
self.begin_time,
conn) for ts in self.ts_ids}
conn.close()
self.vehicles = dict()
self.reward_range = (-float('inf'), float('inf'))
self.metadata = {}
self.spec = EnvSpec('SUMORL-v0')
self.run = 0
self.metrics = []
self.out_csv_name = out_csv_name
self.observations = {ts: None for ts in self.ts_ids}
self.rewards = {ts: None for ts in self.ts_ids}
def _start_simulation(self):
sumo_cmd = [self._sumo_binary,
'-n', self._net,
'-r', self._route,
'--max-depart-delay', str(self.max_depart_delay),
'--waiting-time-memory', '10000',
'--time-to-teleport', str(self.time_to_teleport)]
if self.begin_time > 0:
sumo_cmd.append('-b {}'.format(self.begin_time))
if self.sumo_seed == 'random':
sumo_cmd.append('--random')
else:
sumo_cmd.extend(['--seed', str(self.sumo_seed)])
if not self.sumo_warnings:
sumo_cmd.append('--no-warnings')
if self.use_gui:
sumo_cmd.extend(['--start', '--quit-on-end'])
if self.virtual_display is not None:
sumo_cmd.extend(['--window-size', f'{self.virtual_display[0]},{self.virtual_display[1]}'])
from pyvirtualdisplay.smartdisplay import SmartDisplay
print("Creating a virtual display.")
self.disp = SmartDisplay(size=self.virtual_display)
self.disp.start()
print("Virtual display started.")
if LIBSUMO:
traci.start(sumo_cmd)
self.sumo = traci
else:
traci.start(sumo_cmd, label=self.label)
self.sumo = traci.getConnection(self.label)
if self.use_gui:
self.sumo.gui.setSchema(traci.gui.DEFAULT_VIEW, "real world")
def reset(self, seed: Optional[int] = None, **kwargs):
if self.run != 0:
self.close()
self.save_csv(self.out_csv_name, self.run)
self.run += 1
self.metrics = []
if seed is not None:
self.sumo_seed = seed
self._start_simulation()
self.traffic_signals = {ts: TrafficSignal(self,
ts,
self.delta_time,
self.yellow_time,
self.min_green,
self.max_green,
self.begin_time,
self.sumo) for ts in self.ts_ids}
self.vehicles = dict()
if self.single_agent:
return self._compute_observations()[self.ts_ids[0]]
else:
return self._compute_observations()
@property
def sim_step(self):
"""
Return current simulation second on SUMO
"""
return self.sumo.simulation.getTime()
def step(self, action):
# No action, follow fixed TL defined in self.phases
if action is None or action == {}:
for _ in range(self.delta_time):
self._sumo_step()
else:
self._apply_actions(action)
self._run_steps()
observations = self._compute_observations()
rewards = self._compute_rewards()
dones = self._compute_dones()
info = self._compute_info()
if self.single_agent:
return observations[self.ts_ids[0]], rewards[self.ts_ids[0]], dones['__all__'], info
else:
return observations, rewards, dones, info
def _run_steps(self):
time_to_act = False
while not time_to_act:
self._sumo_step()
for ts in self.ts_ids:
self.traffic_signals[ts].update()
if self.traffic_signals[ts].time_to_act:
time_to_act = True
def _apply_actions(self, actions):
"""
Set the next green phase for the traffic signals
:param actions: If single-agent, actions is an int between 0 and self.num_green_phases (next green phase)
If multiagent, actions is a dict {ts_id : greenPhase}
"""
if self.single_agent:
if self.traffic_signals[self.ts_ids[0]].time_to_act:
self.traffic_signals[self.ts_ids[0]].set_next_phase(actions)
else:
for ts, action in actions.items():
if self.traffic_signals[ts].time_to_act:
self.traffic_signals[ts].set_next_phase(action)
def _compute_dones(self):
dones = {ts_id: False for ts_id in self.ts_ids}
dones['__all__'] = self.sim_step > self.sim_max_time
return dones
def _compute_info(self):
info = self._compute_step_info()
self.metrics.append(info)
return info
def _compute_observations(self):
self.observations.update({ts: self.traffic_signals[ts].compute_observation() for ts in self.ts_ids if self.traffic_signals[ts].time_to_act})
return {ts: self.observations[ts].copy() for ts in self.observations.keys() if self.traffic_signals[ts].time_to_act}
def _compute_rewards(self):
self.rewards.update({ts: self.traffic_signals[ts].compute_reward() for ts in self.ts_ids if self.traffic_signals[ts].time_to_act})
return {ts: self.rewards[ts] for ts in self.rewards.keys() if self.traffic_signals[ts].time_to_act}
@property
def observation_space(self):
return self.traffic_signals[self.ts_ids[0]].observation_space
@property
def action_space(self):
return self.traffic_signals[self.ts_ids[0]].action_space
def observation_spaces(self, ts_id):
return self.traffic_signals[ts_id].observation_space
def action_spaces(self, ts_id):
return self.traffic_signals[ts_id].action_space
def _sumo_step(self):
self.sumo.simulationStep()
def _compute_step_info(self):
return {
'step_time': self.sim_step,
'reward': self.traffic_signals[self.ts_ids[0]].last_reward,
'total_stopped': sum(self.traffic_signals[ts].get_total_queued() for ts in self.ts_ids),
'total_wait_time': sum(sum(self.traffic_signals[ts].get_waiting_time_per_lane()) for ts in self.ts_ids)
}
def close(self):
if self.sumo is None:
return
if not LIBSUMO:
traci.switch(self.label)
traci.close()
try:
self.disp.stop()
except AttributeError:
pass
self.sumo = None
def __del__(self):
self.close()
def render(self, mode='human'):
if self.virtual_display:
#img = self.sumo.gui.screenshot(traci.gui.DEFAULT_VIEW,
# f"temp/img{self.sim_step}.jpg",
# width=self.virtual_display[0],
# height=self.virtual_display[1])
img = self.disp.grab()
if mode == 'rgb_array':
return np.array(img)
return img
def save_csv(self, out_csv_name, run):
if out_csv_name is not None:
df = pd.DataFrame(self.metrics)
Path(Path(out_csv_name).parent).mkdir(parents=True, exist_ok=True)
df.to_csv(out_csv_name + '_conn{}_run{}'.format(self.label, run) + '.csv', index=False)
# Below functions are for discrete state space
def encode(self, state, ts_id):
phase = int(np.where(state[:self.traffic_signals[ts_id].num_green_phases] == 1)[0])
min_green = state[self.traffic_signals[ts_id].num_green_phases]
density_queue = [self._discretize_density(d) for d in state[self.traffic_signals[ts_id].num_green_phases + 1:]]
# tuples are hashable and can be used as key in python dictionary
return tuple([phase, min_green] + density_queue)
def _discretize_density(self, density):
return min(int(density*10), 9)
class Hangouts(object):
"""
Main class for controlling hangout calls.
Initialization does two things:
1. Makes sure that there is active X session.
2. Starts the browser.
If 'DISPLAY' can't be found in os.environ than new X session starts.
Starting new session handels `PyVirtualDisplay`_.
.. _PyVirtualDisplay: http://ponty.github.io/PyVirtualDisplay/
For handling browser used seleniumwrapper library.
.. testsetup:: HangoutsBase
import os
os.environ['DISPLAY'] = '1'
from hangout_api import Hangouts
from hangout_api.tests.doctests_utils import DummySelenium
import seleniumwrapper
def get_attribute(self, name):
if name == 'aria-label':
return ' John Doe '
elif name == 'data-userid':
return '108775712935'
else:
return 'hello'
DummySelenium.get_attribute = get_attribute
seleniumwrapper.create = DummySelenium
.. testsetup:: HangoutsBase2
import os
os.environ['DISPLAY'] = '1'
from hangout_api import Hangouts
from hangout_api.tests.doctests_utils import DummySelenium
import seleniumwrapper
DummySelenium.location = {'x': 1}
seleniumwrapper.create = DummySelenium
hangout = Hangouts()
.. doctest:: HangoutsBase
>>> hangout = Hangouts()
"""
def __init__(self, executable_path=None, chrome_options=None):
self.hangout_id = None
self.on_air = None
# lets start display in case if no is available
self.display = None
if not os.environ.get('DISPLAY'):
self.display = SmartDisplay()
self.display.start()
kwargs = {'executable_path': executable_path or CHROMEDRV_PATH}
if chrome_options is not None:
kwargs['chrome_options'] = chrome_options
self.browser = selwrap.create('chrome', **kwargs)
self.utils = Utils(self.browser)
for name, instance in getUtilitiesFor(IModule):
setattr(self, name, instance(self.utils))
def start(self, on_air=None):
"""
Start a new hangout.
After new hangout is created its id is stored in 'hangout_id' attribure
.. doctest:: HangoutsBase
>>> hangout.start()
>>> hangout.hangout_id
'gs4pp6g62w65moctfqsvihzq2qa'
To start OnAir just pass on_air argument to 'start' method.
.. doctest:: HangoutsBase
>>> hangout.start(
... on_air={'name':'My OnAir', 'attendees':['Friends']})
>>> hangout.start(on_air='https://plus.google.com/events/df34...')
"""
# onair
if on_air is not None:
self.on_air = on_air
if isinstance(on_air, dict):
# in case if on_air is a dict create new hangout event
_create_hangout_event(
self.browser, on_air['name'], on_air['attendees'])
else:
# otherwise (hangout is a string) go to event page
self.browser.get(on_air)
# on event page, redirecting can take some time
self.browser.xpath(
'//div[@data-tooltip="Start the Hangout On Air"]',
timeout=TIMEOUTS.extralong).click(TIMEOUTS.fast)
for name, instance in getUtilitiesFor(IOnAirModule):
setattr(self, name, instance(self.utils))
else:
if not self.browser.current_url.startswith(
URLS.hangouts_active_list):
self.browser.get(URLS.hangouts_active_list)
# G+ opens new window for new hangout, so we need to
# switch selenium to it
self.browser.by_text(
'Start a video Hangout').click(timeout=TIMEOUTS.fast)
# waiting until new window appears
tries_n_time_until_true(
lambda: len(self.browser.window_handles) <= 1, try_num=100)
# self.browser.close() # closing old window
self.browser.switch_to_window(self.browser.window_handles[-1])
self.utils.click_cancel_button_if_there_is_one(
timeout=TIMEOUTS.extralong)
if self.on_air:
# waiting for broadcasting to be ready
broadcast_button = self.browser.by_text(
'Start broadcast', timeout=TIMEOUTS.long)
tries_n_time_until_true(broadcast_button.is_displayed, try_num=600)
# setting hangout id property
self.hangout_id = self.browser.current_url.replace(
URLS.hangout_session_base, '', 1).split('?', 1)[0]
# @retry(stop_max_attempt_number=3)
def connect(self, hangout_id):
"""
Connect to an existing hangout.
Takes id of targeted hangout as argument.
Also it sets hangout_id property:
.. doctest:: HangoutsBase
>>> hangout.connect('fnar4989hf9834h')
>>> hangout.hangout_id
'fnar4989hf9834h'
"""
self.hangout_id = hangout_id
self.browser.get(URLS.hangout_session_base + hangout_id)
# there may be a big delay before 'Join' button appears, so there is a
# need wait longer than usual
join_button = self.browser.xpath(
'//*[text()="Join" or text()="Okay, got it!"]',
timeout=TIMEOUTS.long)
button_text = names_cleaner(join_button.get_attribute('innerText'))
if button_text == 'Okay, got it!':
# to join hangout we need to set agreement checkbox
self.browser.xpath('//*[@role="presentation"]').click(
timeout=TIMEOUTS.fast)
join_button.click(timeout=TIMEOUTS.average)
self.browser.by_text('Join', timeout=TIMEOUTS.long).click(
timeout=TIMEOUTS.fast)
@retry(stop_max_attempt_number=3)
def login(self, username, password, otp=None):
"""
Log in to google plus.
*otp* argument is one time password and it's optional,
set it only if you're using 2-factor authorization.
.. doctest:: HangoutsBase
>>> hangout.login('*****@*****.**', 'password')
>>> hangout.login('*****@*****.**', 'password', otp='123456')
"""
# Open login form and sing in with credentials
self.browser.get(URLS.service_login)
self.browser.by_id('Email').send_keys(username)
self.browser.by_id('Passwd').send_keys(password)
self.browser.by_id('signIn').click(timeout=TIMEOUTS.fast)
# filling up one time password if provides
if otp:
self.browser.by_id('smsUserPin').send_keys(otp)
self.browser.by_id('smsVerifyPin').click(timeout=TIMEOUTS.fast)
# checking if log in was successful
if not self.utils.is_logged_in:
raise LoginError(
'Wasn\'t able to login. Check if credentials are correct'
'and make sure that you have G+ account activated')
@retry(stop_max_attempt_number=3)
def invite(self, participants):
"""
Invite person or circle to hangout:
.. doctest:: HangoutsBase2
>>> hangout.invite("*****@*****.**")
>>> hangout.invite(["*****@*****.**", "Public", "Friends"])
"""
self.utils.click_cancel_button_if_there_is_one()
if not any(isinstance(participants, i) for i in (list, tuple)):
participants = [participants, ]
# click on Invite People button
self.utils.click_menu_element('//div[@aria-label="Invite People"]')
input_field = self.browser.xpath(
'//input[@placeholder="+ Add names, circles, or email addresses"]')
input_field.click()
input_field.clear()
for participant in participants:
input_field.send_keys(participant)
sleep(1) # need to wait a bit for HG to make request
input_field.send_keys(Keys.RETURN)
self.browser.by_text('Invite').click(timeout=TIMEOUTS.fast)
# making sure that invitation is posted
xpath = '//*[text()="Invitation posted"'\
'or text()="Waiting for people to join this video call..."]'
self.browser.xpath(xpath)
@retry(stop_max_attempt_number=3)
def participants(self):
"""
Returns list of namedtuples of current participants:
.. doctest:: HangoutsBase
>>> hangout.participants()
[Participant(name='John Doe', profile_id='108775712935')]
"""
xpath = '//div[@data-userid]'
# for some reason some persons can be listed twice, so let's
# filter them with dict
participants = {
p.get_attribute('data-userid'): p.get_attribute(
'aria-label')[5:-11]
for p in self.browser.xpath(xpath, eager=True)}
return [Participant(name=pname, profile_id=pid)
for pid, pname in participants.items()]
@retry(stop_max_attempt_number=3)
def disconnect(self):
"""
Leave hangout (equal on clicking on "Leave call" button). After
leaving the call you can create a new one or connect to existing.
.. doctest:: HangoutsBase2
>>> hangout.disconnect()
"""
self.utils.click_cancel_button_if_there_is_one()
self.utils.click_menu_element('//div[@aria-label="Leave call"]')
self.hangout_id = None
if self.on_air is not None:
# removing properties that is available only for OnAir
for name, _ in getUtilitiesFor(IOnAirModule):
delattr(self, name)
self.on_air = None
# waiting until hangout windows closes
tries_n_time_until_true(
lambda: len(self.browser.window_handles) == 1, try_num=100)
self.browser.switch_to_window(self.browser.window_handles[-1])
def __del__(self):
# and quiting browser and display
if self.browser:
try:
if self.hangout_id and self.browser.window_handles:
# leaving the call first
self.disconnect()
finally:
self.browser.quit()
if self.display is not None:
self.display.stop()
class SeleniumExecutor(ScenarioExecutor, WidgetProvider, FileLister):
"""
Selenium executor
:type virtual_display: Display
:type runner: AbstractTestRunner
"""
SELENIUM_DOWNLOAD_LINK = "http://selenium-release.storage.googleapis.com/{version}/" \
"selenium-server-standalone-{version}.0.jar"
SELENIUM_VERSION = "2.53"
JUNIT_DOWNLOAD_LINK = "http://search.maven.org/remotecontent?filepath=junit/junit/" \
"{version}/junit-{version}.jar"
JUNIT_VERSION = "4.12"
JUNIT_MIRRORS_SOURCE = "http://search.maven.org/solrsearch/select?q=g%3A%22junit%22%20AND%20a%3A%22" \
"junit%22%20AND%20v%3A%22{version}%22&rows=20&wt=json".format(version=JUNIT_VERSION)
HAMCREST_DOWNLOAD_LINK = "http://search.maven.org/remotecontent?filepath=org/hamcrest/hamcrest-core" \
"/1.3/hamcrest-core-1.3.jar"
JSON_JAR_DOWNLOAD_LINK = "http://search.maven.org/remotecontent?filepath=org/json/json/20160810/json-20160810.jar"
SUPPORTED_TYPES = [".py", ".jar", ".java"]
SHARED_VIRTUAL_DISPLAY = {}
def __init__(self):
super(SeleniumExecutor, self).__init__()
self.additional_env = {}
self.virtual_display = None
self.end_time = None
self.runner = None
self.reader = None
self.report_file = None
self.runner_working_dir = None
self.scenario = None
self.script = None
self.self_generated_script = False
self.generated_methods = BetterDict()
def set_virtual_display(self):
display_conf = self.settings.get("virtual-display")
if display_conf:
if is_windows():
self.log.warning(
"Cannot have virtual display on Windows, ignoring")
else:
if self.engine in SeleniumExecutor.SHARED_VIRTUAL_DISPLAY:
self.virtual_display = SeleniumExecutor.SHARED_VIRTUAL_DISPLAY[
self.engine]
else:
width = display_conf.get("width", 1024)
height = display_conf.get("height", 768)
self.virtual_display = Display(size=(width, height))
msg = "Starting virtual display[%s]: %s"
self.log.info(msg, self.virtual_display.size,
self.virtual_display.new_display_var)
self.virtual_display.start()
SeleniumExecutor.SHARED_VIRTUAL_DISPLAY[
self.engine] = self.virtual_display
def free_virtual_display(self):
if self.virtual_display and self.virtual_display.is_alive():
self.virtual_display.stop()
if self.engine in SeleniumExecutor.SHARED_VIRTUAL_DISPLAY:
del SeleniumExecutor.SHARED_VIRTUAL_DISPLAY[self.engine]
def get_script_path(self, scenario=None):
if scenario:
return super(SeleniumExecutor, self).get_script_path(scenario)
else:
return self.engine.find_file(self.script)
def _create_runner(self, working_dir, report_file):
script_path = self.get_script_path()
script_type = self.detect_script_type(script_path)
runner_config = BetterDict()
if script_type == ".py":
runner_class = NoseTester
runner_config.merge(
self.settings.get("selenium-tools").get("nose"))
else: # script_type == ".jar" or script_type == ".java":
runner_class = JUnitTester
runner_config.merge(
self.settings.get("selenium-tools").get("junit"))
runner_config['props-file'] = self.engine.create_artifact(
"customrunner", ".properties")
runner_config["script-type"] = script_type
runner_config["working-dir"] = working_dir
runner_config.get("artifacts-dir", self.engine.artifacts_dir)
runner_config.get("report-file", report_file)
runner_config.get("stdout",
self.engine.create_artifact("junit", ".out"))
runner_config.get("stderr",
self.engine.create_artifact("junit", ".err"))
return runner_class(runner_config, self)
def _create_reader(self, report_file):
return LoadSamplesReader(report_file, self.log, self.generated_methods)
def prepare(self):
self.set_virtual_display()
self.scenario = self.get_scenario()
self._verify_script()
self.runner_working_dir = self.engine.create_artifact("classes", "")
self.report_file = self.engine.create_artifact("selenium_tests_report",
".ldjson")
self.runner = self._create_runner(self.runner_working_dir,
self.report_file)
self._cp_resource_files(self.runner_working_dir)
self.runner.prepare()
self.reader = self._create_reader(self.report_file)
if isinstance(self.engine.aggregator, ConsolidatingAggregator):
self.engine.aggregator.add_underling(self.reader)
def _verify_script(self):
if Scenario.SCRIPT in self.scenario and self.scenario.get(
Scenario.SCRIPT):
self.script = self.scenario.get(Scenario.SCRIPT)
elif "requests" in self.scenario:
self.script = self.__tests_from_requests()
self.self_generated_script = True
else:
raise ValueError(
"Nothing to test, no requests were provided in scenario")
def _cp_resource_files(self, runner_working_dir):
script = self.get_script_path()
if os.path.isdir(script):
shutil.copytree(script, runner_working_dir)
else:
os.makedirs(runner_working_dir)
if self.self_generated_script:
shutil.move(script, runner_working_dir)
else:
script_type = self.detect_script_type(script)
script_name = os.path.basename(script)
if script_type == ".py" and not script_name.lower().startswith(
'test'):
target_name = 'test_' + script_name
msg = "Script '%s' won't be discovered by nosetests, renaming script to %s"
self.log.warning(msg, script_name, target_name)
else:
target_name = script_name
target_path = os.path.join(runner_working_dir, target_name)
shutil.copy2(script, target_path)
@staticmethod
def detect_script_type(script_path):
if not isinstance(script_path, string_types) and not isinstance(
script_path, text_type):
raise ValueError(
"Nothing to test, no files were provided in scenario")
if not os.path.exists(script_path):
raise ValueError("Script %s doesn't exist" % script_path)
file_types = set()
if os.path.isfile(script_path): # regular file received
file_types.add(os.path.splitext(script_path)[1].lower())
else: # dir received: check contained files
for file_name in get_files_recursive(script_path):
file_types.add(os.path.splitext(file_name)[1].lower())
if '.java' in file_types:
file_ext = '.java'
elif '.py' in file_types:
file_ext = '.py'
elif '.jar' in file_types:
file_ext = '.jar'
else:
raise ValueError("Unsupported script type: %s" % script_path)
return file_ext
def startup(self):
"""
Start runner
:return:
"""
self.start_time = time.time()
self.runner.env = self.additional_env
self.runner.run_tests()
def check_virtual_display(self):
if self.virtual_display:
if not self.virtual_display.is_alive():
self.log.info("Virtual display out: %s",
self.virtual_display.stdout)
self.log.warning("Virtual display err: %s",
self.virtual_display.stderr)
raise RuntimeError("Virtual display failed: %s" %
self.virtual_display.return_code)
def check(self):
"""
check if test completed
:return:
"""
if self.widget:
self.widget.update()
self.check_virtual_display()
return self.runner.is_finished()
def report_test_duration(self):
if self.start_time:
self.end_time = time.time()
self.log.debug("Selenium tests ran for %s seconds",
self.end_time - self.start_time)
def shutdown(self):
"""
shutdown test_runner
:return:
"""
self.runner.shutdown()
self.report_test_duration()
def post_process(self):
self.free_virtual_display()
if self.reader and not self.reader.read_records:
raise RuntimeWarning("Empty results, most likely Selenium failed")
def get_widget(self):
if not self.widget:
self.widget = SeleniumWidget(self.script,
self.runner.settings.get("stdout"))
return self.widget
def resource_files(self):
self.scenario = self.get_scenario()
self._verify_script()
script_path = self.get_script_path()
resources = []
if script_path is not None:
resources.append(script_path)
return resources
def __tests_from_requests(self):
filename = self.engine.create_artifact("test_requests", ".py")
wdlog = self.engine.create_artifact('webdriver', '.log')
nose_test = SeleniumScriptBuilder(self.scenario, self.log, wdlog)
if self.virtual_display:
nose_test.window_size = self.virtual_display.size
self.generated_methods.merge(nose_test.gen_test_case())
nose_test.save(filename)
return filename
from easyprocess import EasyProcess
import time
import json
from io import BytesIO
handler = logging.StreamHandler(sys.stdout)
formatter = logging.Formatter(
"%(asctime)s : %(levelname)s : %(name)s : %(message)s")
handler.setFormatter(formatter)
logger = logging.getLogger()
logger.addHandler(handler)
logger.setLevel(logging.INFO)
# Must start display before we can use Xlib via pynput
disp = SmartDisplay(visible=False, size=(240, 160))
disp.start()
logger.info("xvfb display started")
from pynput.keyboard import Key, Controller
logger.info("pynput imported and working")
# Have to start pulse here because mgba is being a weenie
pulse = EasyProcess("pulseaudio")
pulse.start()
logger.info("pulseaudio started")
with open("config.json") as f:
config = json.load(f)
if config["debug"]:
logger.setLevel(logging.DEBUG)
application = server.Application(config["hs_address"], config["token"])
import time
import lxml.html
import pandas as pd
from selenium import webdriver
from pyvirtualdisplay.smartdisplay import SmartDisplay
url = 'https://coronavirus.1point3acres.com/en'
next_page_target = '.active .case-table .ant-pagination-next[aria-disabled=false]'
cols = ['case', 'date', 'state', 'county', 'notes', 'news_reference']
if __name__ == '__main__':
display = SmartDisplay(visible=0, size=(1920, 1080))
display.start()
# Necessary for headless on server
options = webdriver.ChromeOptions()
options.add_argument('--no-sandbox')
driver = webdriver.Chrome(chrome_options=options)
driver.get(url)
page = 1
rows = []
while True:
html = driver.find_element_by_tag_name('html').get_attribute(
'innerHTML')
html = lxml.html.fromstring(html)
table = html.cssselect('.active .case-table')[0]
for row_el in table.cssselect('.ant-table-row'):
row = {}
for name, el in zip(cols, row_el.cssselect('td')):
class SeleniumExecutor(ScenarioExecutor, WidgetProvider, FileLister):
"""
Selenium executor
:type virtual_display: Display
"""
SELENIUM_DOWNLOAD_LINK = "http://selenium-release.storage.googleapis.com/{version}/" \
"selenium-server-standalone-{version}.0.jar"
SELENIUM_VERSION = "2.48"
JUNIT_DOWNLOAD_LINK = "http://search.maven.org/remotecontent?filepath=junit/junit/{version}/junit-{version}.jar"
JUNIT_VERSION = "4.12"
JUNIT_MIRRORS_SOURCE = "http://search.maven.org/solrsearch/select?q=g%3A%22junit%22%20AND%20a%3A%22junit%22%20" \
"AND%20v%3A%22{version}%22&rows=20&wt=json".format(version=JUNIT_VERSION)
HAMCREST_DOWNLOAD_LINK = "https://hamcrest.googlecode.com/files/hamcrest-core-1.3.jar"
SUPPORTED_TYPES = [".py", ".jar", ".java"]
def __init__(self):
super(SeleniumExecutor, self).__init__()
self.virtual_display = None
self.start_time = None
self.end_time = None
self.runner = None
self.widget = None
self.reader = None
self.kpi_file = None
self.err_jtl = None
self.runner_working_dir = None
self.scenario = None
def prepare(self):
display_conf = self.settings.get("virtual-display")
if display_conf:
if is_windows():
self.log.warning("Cannot have virtual display on Windows, ignoring")
else:
width = display_conf.get("width", 1024)
height = display_conf.get("height", 768)
self.virtual_display = Display(size=(width, height))
self.scenario = self.get_scenario()
self._verify_script()
self.kpi_file = self.engine.create_artifact("selenium_tests_report", ".csv")
self.err_jtl = self.engine.create_artifact("selenium_tests_err", ".xml")
script_type = self.detect_script_type(self.scenario.get(Scenario.SCRIPT))
if script_type == ".py":
runner_class = NoseTester
runner_config = self.settings.get("selenium-tools").get("nose")
elif script_type == ".jar" or script_type == ".java":
runner_class = JUnitTester
runner_config = self.settings.get("selenium-tools").get("junit")
runner_config['props-file'] = self.engine.create_artifact("customrunner", ".properties")
else:
raise ValueError("Unsupported script type: %s" % script_type)
runner_config["script-type"] = script_type
self.runner_working_dir = self.engine.create_artifact(runner_config.get("working-dir", "classes"), "")
runner_config["working-dir"] = self.runner_working_dir
runner_config.get("artifacts-dir", self.engine.artifacts_dir)
runner_config.get("working-dir", self.runner_working_dir)
runner_config.get("report-file", self.kpi_file)
runner_config.get("err-file", self.err_jtl)
runner_config.get("stdout", self.engine.create_artifact("junit", ".out"))
runner_config.get("stderr", self.engine.create_artifact("junit", ".err"))
self._cp_resource_files(self.runner_working_dir)
self.runner = runner_class(runner_config, self.scenario, self.get_load(), self.log)
self.runner.prepare()
self.reader = JTLReader(self.kpi_file, self.log, self.err_jtl)
if isinstance(self.engine.aggregator, ConsolidatingAggregator):
self.engine.aggregator.add_underling(self.reader)
def _verify_script(self):
if not self.scenario.get("script"):
if self.scenario.get("requests"):
self.scenario["script"] = self.__tests_from_requests()
else:
raise RuntimeError("Nothing to test, no requests were provided in scenario")
def _cp_resource_files(self, runner_working_dir):
"""
:return:
"""
script = self.scenario.get("script")
if Scenario.SCRIPT in self.scenario:
if os.path.isdir(script):
shutil.copytree(script, runner_working_dir)
else:
os.makedirs(runner_working_dir)
shutil.copy2(script, runner_working_dir)
def detect_script_type(self, script_path):
"""
checks if script is java or python
if it's folder or single script
:return:
"""
if not isinstance(script_path, string_types) and not isinstance(script_path, text_type):
raise RuntimeError("Nothing to test, no files were provided in scenario")
test_files = []
for dir_entry in os.walk(script_path):
if dir_entry[2]:
for test_file in dir_entry[2]:
if os.path.splitext(test_file)[1].lower() in SeleniumExecutor.SUPPORTED_TYPES:
test_files.append(test_file)
if os.path.isdir(script_path):
file_ext = os.path.splitext(test_files[0])[1].lower()
else:
file_ext = os.path.splitext(script_path)[1]
if file_ext not in SeleniumExecutor.SUPPORTED_TYPES:
raise RuntimeError("Supported tests types %s was not found" % SeleniumExecutor.SUPPORTED_TYPES)
return file_ext
def startup(self):
"""
Start runner
:return:
"""
self.start_time = time.time()
if self.virtual_display:
msg = "Starting virtual display[%s]: %s"
self.log.info(msg, self.virtual_display.size, self.virtual_display.new_display_var)
self.virtual_display.start()
self.runner.run_tests()
def check(self):
"""
check if test completed
:return:
"""
if self.widget:
self.widget.update()
if self.virtual_display:
if not self.virtual_display.is_alive():
self.log.info("Virtual display out: %s", self.virtual_display.stdout)
self.log.warning("Virtual display err: %s", self.virtual_display.stderr)
raise RuntimeError("Virtual display failed: %s" % self.virtual_display.return_code)
return self.runner.is_finished()
def shutdown(self):
"""
shutdown test_runner
:return:
"""
try:
self.runner.shutdown()
finally:
if self.virtual_display and self.virtual_display.is_alive():
self.virtual_display.stop()
if self.start_time:
self.end_time = time.time()
self.log.debug("Selenium tests ran for %s seconds", self.end_time - self.start_time)
def post_process(self):
if self.reader and not self.reader.read_records:
raise RuntimeWarning("Empty results, most likely Selenium failed")
def get_widget(self):
if not self.widget:
self.widget = SeleniumWidget(self.scenario.get("script"), self.runner.settings.get("stdout"))
return self.widget
def resource_files(self):
if not self.scenario:
self.scenario = self.get_scenario()
if "script" not in self.scenario:
return []
script = self.scenario.get("script")
script_type = self.detect_script_type(script)
if script_type == ".py":
runner_config = self.settings.get("selenium-tools").get("nose")
elif script_type == ".jar" or script_type == ".java":
runner_config = self.settings.get("selenium-tools").get("junit")
else:
raise ValueError("Unsupported script type: %s" % script_type)
if self.runner_working_dir is None:
self.runner_working_dir = self.engine.create_artifact(runner_config.get("working-dir", "classes"), "")
self._cp_resource_files(self.runner_working_dir)
return [os.path.basename(self.runner_working_dir)]
def __tests_from_requests(self):
filename = self.engine.create_artifact("test_requests", ".py")
nose_test = SeleniumScriptBuilder(self.scenario, self.log)
if self.virtual_display:
nose_test.window_size=self.virtual_display.size
nose_test.scenario = self.scenario
nose_test.gen_test_case()
nose_test.save(filename)
return filename
class BrowserInstance:
def __init__(self,
tor=False,
tor_tries=10,
save_directory="data",
url_delay=10,
random_pages=False,
test=False):
self.tor = tor
self.tor_tries = tor_tries
self.save_directory = save_directory
self.url_delay = url_delay
self.random_pages = random_pages
self.sites = ["amazon", "apple", "nytimes", "google", "duckduckgo"]
pathlib.Path(self.save_directory).mkdir(parents=True, exist_ok=True)
self.display = SmartDisplay(backend='xephyr',
visible=1,
size=(1920, 1080))
self.display.start()
self.firefox_profile_path = tempfile.mkdtemp()
self.firefox_profile = webdriver.FirefoxProfile(
self.firefox_profile_path)
if self.tor:
for i in range(self.tor_tries):
try:
self.socks_port = free_port()
self.control_port = free_port()
self.tor_data_dir = tempfile.mkdtemp()
self.torrc = {
'ControlPort': str(self.control_port),
'SOCKSPort': str(self.socks_port),
'DataDirectory': self.tor_data_dir,
'EntryNodes': '{us}',
'ExitNodes': '{us}',
'StrictNodes': '1'
}
self.tor_process = launch_tor_with_config(
config=self.torrc, tor_cmd="/usr/sbin/tor")
except:
print("Tor connection attempt {} failed.".format(i))
sys.stdout.flush()
if i == self.tor_tries - 1:
sys.exit()
continue
break
self.firefox_profile.set_preference("network.proxy.type", 1)
self.firefox_profile.set_preference("network.proxy.socks",
"localhost")
self.firefox_profile.set_preference("network.proxy.socks_port",
self.socks_port)
self.firefox_profile.set_preference("network.proxy.socks_version",
5)
self.firefox_profile.update_preferences()
self.driver = webdriver.Firefox(service_log_path=os.path.join(
self.firefox_profile_path, "geckodriver.log"),
firefox_profile=self.firefox_profile)
self.driver.maximize_window()
if test:
self.driver.get("https://www.{}.com/".format(
random.choice(self.sites)))
time.sleep(self.url_delay)
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
self.cleanup()
def cleanup(self):
self.display.stop()
shutil.rmtree(self.firefox_profile_path)
if self.tor:
self.tor_process.kill()
shutil.rmtree(self.tor_data_dir)
def print_ports(self):
print("SOCKS Port: {}, Control Port: {}".format(
self.socks_port, self.control_port))
def tar_and_hash(self, contents):
tmp_tar_directory = tempfile.mkdtemp()
archive_name = os.path.join(tmp_tar_directory, "tarball.txz")
with tarfile.open(archive_name, "w:xz") as tar:
tar.add(contents)
h = hashlib.sha256()
with open(archive_name, 'rb') as f:
h.update(f.read())
archive_hash = h.hexdigest()
shutil.copy(
archive_name,
os.path.join(self.save_directory, "{}.txz".format(archive_hash)))
shutil.rmtree(tmp_tar_directory)
return archive_hash
def save_html(self):
fd, path = tempfile.mkstemp()
with open(path, "w") as f:
f.write(self.driver.page_source)
os.close(fd)
archive_hash = self.tar_and_hash(path)
os.remove(path)
return archive_hash
def save_page(self):
pyautogui._pyautogui_x11._display = Xlib.display.Display(
os.environ['DISPLAY'])
tmp_page_directory = tempfile.mkdtemp()
pyautogui.PAUSE = self.url_delay
pyautogui.hotkey('alt', 'tab')
pyautogui.hotkey('ctrl', 's')
pyautogui.write(os.path.join(tmp_page_directory, "page"))
pyautogui.press('enter')
time.sleep(self.url_delay)
archive_hash = self.tar_and_hash(tmp_page_directory)
shutil.rmtree(tmp_page_directory)
return archive_hash
def scroll_page(self):
height = 0
height_increment = 500
total_height = int(
self.driver.execute_script("return document.body.scrollHeight"))
while (height <= total_height):
self.driver.execute_script("window.scrollTo({}, {});".format(
height, height + height_increment))
time.sleep(2)
height += height_increment
total_height = int(
self.driver.execute_script(
"return document.body.scrollHeight"))
def visit_random_page(self):
if self.random_pages and random.choices([True, False],
[0.25, 0.75])[0]:
self.driver.get("https://www.{}.com/".format(
random.choice(self.sites)))
time.sleep(self.url_delay)
def get_url(self, url):
try:
self.visit_random_page()
self.driver.get(url)
time.sleep(self.url_delay)
self.scroll_page()
self.url_tries_cound = 0
return BeautifulSoup(self.driver.page_source, "html.parser")
except:
print("URL Connection Error: {}".format(url))
sys.stdout.flush()
return False
def get_page_from_archive(self, archive_file):
tmp_page_directory = tempfile.mkdtemp()
os.chdir(tmp_page_directory)
with tarfile.open(archive_file, "r:xz") as tar:
tar.extractall(tmp_page_directory)
url = glob.glob('**/page.html', recursive=True)
if url:
url = "file://" + tmp_page_directory + "/" + url[0]
self.driver.get(url)
s = BeautifulSoup(self.driver.page_source, "html.parser")
else:
s = False
os.chdir(self.save_directory)
shutil.rmtree(tmp_page_directory)
return s
class VirtualDisplay(Service, Singletone):
"""
Selenium executor
:type virtual_display: Display
"""
SHARED_VIRTUAL_DISPLAY = {}
def __init__(self):
super(VirtualDisplay, self).__init__()
self.virtual_display = None
def get_virtual_display(self):
return self.virtual_display
def set_virtual_display(self):
if is_windows():
self.log.warning("Cannot have virtual display on Windows, ignoring")
return
if self.engine in VirtualDisplay.SHARED_VIRTUAL_DISPLAY:
self.virtual_display = VirtualDisplay.SHARED_VIRTUAL_DISPLAY[self.engine]
else:
width = self.parameters.get("width", 1024)
height = self.parameters.get("height", 768)
self.virtual_display = Display(size=(width, height))
msg = "Starting virtual display[%s]: %s"
self.log.info(msg, self.virtual_display.size, self.virtual_display.new_display_var)
self.virtual_display.start()
# roll DISPLAY back for online report browser
if self.virtual_display.old_display_var:
os.environ["DISPLAY"] = self.virtual_display.old_display_var
else:
del os.environ["DISPLAY"]
VirtualDisplay.SHARED_VIRTUAL_DISPLAY[self.engine] = self.virtual_display
self.engine.shared_env.set({"DISPLAY": self.virtual_display.new_display_var})
def free_virtual_display(self):
if self.virtual_display and self.virtual_display.is_alive():
os.environ["DISPLAY"] = self.virtual_display.new_display_var
self.virtual_display.stop()
if self.engine in VirtualDisplay.SHARED_VIRTUAL_DISPLAY:
del VirtualDisplay.SHARED_VIRTUAL_DISPLAY[self.engine]
self.engine.shared_env.set({"DISPLAY": None})
def startup(self):
self.set_virtual_display()
def check_virtual_display(self):
if self.virtual_display:
if not self.virtual_display.is_alive():
self.log.info("Virtual display out: %s", self.virtual_display.stdout)
self.log.warning("Virtual display err: %s", self.virtual_display.stderr)
raise TaurusInternalException("Virtual display failed: %s" % self.virtual_display.return_code)
def check(self):
self.check_virtual_display()
return False
def shutdown(self):
self.free_virtual_display()
class SeleniumExecutor(AbstractSeleniumExecutor, WidgetProvider, FileLister):
"""
Selenium executor
:type virtual_display: Display
:type runner: SubprocessedExecutor
"""
SUPPORTED_RUNNERS = ["nose", "junit", "testng", "rspec", "mocha"]
def __init__(self):
super(SeleniumExecutor, self).__init__()
self.additional_env = {}
self.virtual_display = None
self.end_time = None
self.runner = None
self.script = None
self.generated_methods = BetterDict()
self.runner_working_dir = None
def get_virtual_display(self):
return self.virtual_display
def add_env(self, env):
self.additional_env.update(env)
def set_virtual_display(self):
display_conf = self.settings.get("virtual-display")
if display_conf:
if is_windows():
self.log.warning(
"Cannot have virtual display on Windows, ignoring")
else:
if self.engine in SeleniumExecutor.SHARED_VIRTUAL_DISPLAY:
self.virtual_display = SeleniumExecutor.SHARED_VIRTUAL_DISPLAY[
self.engine]
else:
width = display_conf.get("width", 1024)
height = display_conf.get("height", 768)
self.virtual_display = Display(size=(width, height))
msg = "Starting virtual display[%s]: %s"
self.log.info(msg, self.virtual_display.size,
self.virtual_display.new_display_var)
self.virtual_display.start()
SeleniumExecutor.SHARED_VIRTUAL_DISPLAY[
self.engine] = self.virtual_display
def free_virtual_display(self):
if self.virtual_display and self.virtual_display.is_alive():
self.virtual_display.stop()
if self.engine in SeleniumExecutor.SHARED_VIRTUAL_DISPLAY:
del SeleniumExecutor.SHARED_VIRTUAL_DISPLAY[self.engine]
def get_runner_working_dir(self):
if self.runner_working_dir is None:
self.runner_working_dir = self.engine.create_artifact(
"classes", "")
return self.runner_working_dir
def create_runner(self):
runner_type = self.get_runner_type()
self.runner = self.engine.instantiate_module(runner_type)
# todo: deprecated, remove it later
self.runner.settings.merge(
self.settings.get('selenium-tools').get(runner_type))
self.runner.parameters = self.parameters
self.runner.provisioning = self.provisioning
self.runner.execution = self.execution
self.runner.execution['executor'] = runner_type
if runner_type == "nose":
self.runner.execution["test-mode"] = "selenium"
def prepare(self):
if self.get_load().concurrency and self.get_load().concurrency > 1:
msg = 'Selenium supports concurrency in cloud provisioning mode only\n'
msg += 'For details look at http://gettaurus.org/docs/Cloud.md'
self.log.warning(msg)
self.set_virtual_display()
self.create_runner()
self.runner.prepare()
self.script = self.runner.script
def get_runner_type(self):
if "runner" in self.execution:
runner = self.execution["runner"]
if runner not in SeleniumExecutor.SUPPORTED_RUNNERS:
msg = "Runner '%s' is not supported. Supported runners: %s"
raise TaurusConfigError(
msg % (runner, SeleniumExecutor.SUPPORTED_RUNNERS))
self.log.debug("Using script type: %s", runner)
return runner
script_name = self.get_script_path()
if script_name:
return self.detect_script_type(script_name)
else:
if "requests" in self.get_scenario():
return "nose"
else:
raise TaurusConfigError(
"You must specify either script or list of requests to run Selenium"
)
def resource_files(self):
self.create_runner()
return self.runner.resource_files()
def detect_script_type(self, script_name):
if not os.path.exists(script_name):
raise TaurusConfigError("Script '%s' doesn't exist" % script_name)
file_types = set()
# gather file extensions and choose script_type according to priority
if os.path.isfile(script_name): # regular file received
file_types.add(os.path.splitext(script_name)[1].lower())
else: # dir received: check contained files
for file_name in get_files_recursive(script_name):
file_types.add(os.path.splitext(file_name)[1].lower())
if '.java' in file_types or '.jar' in file_types:
# todo: next detection logic is duplicated in TestNGTester - can we avoid it?
script_dir = get_full_path(self.get_script_path(), step_up=1)
if os.path.exists(os.path.join(
script_dir, 'testng.xml')) or self.execution.get(
'testng-xml', None):
script_type = 'testng'
else:
script_type = 'junit'
elif '.py' in file_types:
script_type = 'nose'
elif '.rb' in file_types:
script_type = 'rspec'
elif '.js' in file_types:
script_type = 'mocha'
else:
raise TaurusConfigError(
"Supported script files not found, script detection is failed")
self.log.debug("Detected script type: %s", script_type)
return script_type
def startup(self):
"""
Start runner
:return:
"""
self.start_time = time.time()
self.runner.env = self.additional_env
self.runner.startup()
def check_virtual_display(self):
if self.virtual_display:
if not self.virtual_display.is_alive():
self.log.info("Virtual display out: %s",
self.virtual_display.stdout)
self.log.warning("Virtual display err: %s",
self.virtual_display.stderr)
raise TaurusInternalException("Virtual display failed: %s" %
self.virtual_display.return_code)
def check(self):
"""
check if test completed
:return:
"""
if self.widget:
self.widget.update()
self.check_virtual_display()
return self.runner.check()
def report_test_duration(self):
if self.start_time:
self.end_time = time.time()
self.log.debug("Selenium tests ran for %s seconds",
self.end_time - self.start_time)
def shutdown(self):
"""
shutdown test_runner
:return:
"""
self.runner.shutdown()
self.report_test_duration()
def post_process(self):
if os.path.exists("geckodriver.log"):
self.engine.existing_artifact("geckodriver.log", True)
self.free_virtual_display()
def has_results(self):
return self.runner.has_results()
def get_widget(self):
if not self.widget:
self.widget = SeleniumWidget(self.script, self.runner.stdout_file)
return self.widget
class SeleniumExecutor(AbstractSeleniumExecutor, WidgetProvider, FileLister):
"""
Selenium executor
:type virtual_display: Display
:type runner: SubprocessedExecutor
"""
SUPPORTED_RUNNERS = ["nose", "junit", "testng", "rspec", "mocha"]
def __init__(self):
super(SeleniumExecutor, self).__init__()
self.additional_env = {}
self.virtual_display = None
self.end_time = None
self.runner = None
self.report_file = None
self.scenario = None
self.script = None
self.generated_methods = BetterDict()
self.runner_working_dir = None
self.register_reader = True
def get_virtual_display(self):
return self.virtual_display
def add_env(self, env):
self.additional_env.update(env)
def set_virtual_display(self):
display_conf = self.settings.get("virtual-display")
if display_conf:
if is_windows():
self.log.warning("Cannot have virtual display on Windows, ignoring")
else:
if self.engine in SeleniumExecutor.SHARED_VIRTUAL_DISPLAY:
self.virtual_display = SeleniumExecutor.SHARED_VIRTUAL_DISPLAY[self.engine]
else:
width = display_conf.get("width", 1024)
height = display_conf.get("height", 768)
self.virtual_display = Display(size=(width, height))
msg = "Starting virtual display[%s]: %s"
self.log.info(msg, self.virtual_display.size, self.virtual_display.new_display_var)
self.virtual_display.start()
SeleniumExecutor.SHARED_VIRTUAL_DISPLAY[self.engine] = self.virtual_display
def free_virtual_display(self):
if self.virtual_display and self.virtual_display.is_alive():
self.virtual_display.stop()
if self.engine in SeleniumExecutor.SHARED_VIRTUAL_DISPLAY:
del SeleniumExecutor.SHARED_VIRTUAL_DISPLAY[self.engine]
def get_runner_working_dir(self):
if self.runner_working_dir is None:
self.runner_working_dir = self.engine.create_artifact("classes", "")
return self.runner_working_dir
def _get_testng_xml(self):
if 'testng-xml' in self.scenario:
testng_xml = self.scenario.get('testng-xml')
if testng_xml:
return testng_xml
else:
return None # empty value for switch off testng.xml path autodetect
script_path = self.get_script_path()
if script_path:
script_dir = get_full_path(script_path, step_up=1)
testng_xml = os.path.join(script_dir, 'testng.xml')
if os.path.exists(testng_xml):
self.log.info("Detected testng.xml file at %s", testng_xml)
self.scenario['testng-xml'] = testng_xml
return testng_xml
return None
def _create_runner(self, report_file):
script_type = self.detect_script_type()
runner = self.engine.instantiate_module(script_type)
runner.parameters = self.parameters
runner.provisioning = self.provisioning
runner.execution = self.execution
runner.execution['executor'] = script_type
if script_type == "nose":
runner.generated_methods = self.generated_methods
runner.execution["test-mode"] = "selenium"
elif script_type == "junit":
pass
elif script_type == "testng":
testng_config = self._get_testng_xml()
if testng_config:
runner.execution['testng-xml'] = self.engine.find_file(testng_config)
elif script_type == "rspec":
pass
elif script_type == "mocha":
pass
else:
raise TaurusConfigError("Unsupported script type: %s" % script_type)
runner.execution["report-file"] = report_file # TODO: shouldn't it be the field?
return runner
def _register_reader(self, report_file):
if self.engine.is_functional_mode():
reader = FuncSamplesReader(report_file, self.engine, self.log, self.generated_methods)
if isinstance(self.engine.aggregator, FunctionalAggregator):
self.engine.aggregator.add_underling(reader)
else:
reader = LoadSamplesReader(report_file, self.log, self.generated_methods)
if isinstance(self.engine.aggregator, ConsolidatingAggregator):
self.engine.aggregator.add_underling(reader)
return reader
def prepare(self):
if self.get_load().concurrency and self.get_load().concurrency > 1:
msg = 'Selenium supports concurrency in cloud provisioning mode only\n'
msg += 'For details look at http://gettaurus.org/docs/Cloud.md'
self.log.warning(msg)
self.set_virtual_display()
self.scenario = self.get_scenario()
self.script = self.get_script_path()
self.report_file = self.engine.create_artifact("selenium_tests_report", ".ldjson")
self.runner = self._create_runner(self.report_file)
self.runner.prepare()
if isinstance(self.runner, NoseTester):
self.script = self.runner._script
if self.register_reader:
self.reader = self._register_reader(self.report_file)
def detect_script_type(self):
if not self.script and "requests" in self.scenario:
return "nose"
if not os.path.exists(self.script):
raise TaurusConfigError("Script '%s' doesn't exist" % self.script)
if "runner" in self.execution:
runner = self.execution["runner"]
if runner not in SeleniumExecutor.SUPPORTED_RUNNERS:
msg = "Runner '%s' is not supported. Supported runners: %s"
raise TaurusConfigError(msg % (runner, SeleniumExecutor.SUPPORTED_RUNNERS))
self.log.debug("Using script type: %s", runner)
return runner
file_types = set()
if os.path.isfile(self.script): # regular file received
file_types.add(os.path.splitext(self.script)[1].lower())
else: # dir received: check contained files
for file_name in get_files_recursive(self.script):
file_types.add(os.path.splitext(file_name)[1].lower())
if '.java' in file_types or '.jar' in file_types:
if self._get_testng_xml() is not None:
script_type = 'testng'
else:
script_type = 'junit'
elif '.py' in file_types:
script_type = 'nose'
elif '.rb' in file_types:
script_type = 'rspec'
elif '.js' in file_types:
script_type = 'mocha'
else:
raise TaurusConfigError("Unsupported script type: %s" % self.script)
self.log.debug("Detected script type: %s", script_type)
return script_type
def startup(self):
"""
Start runner
:return:
"""
self.start_time = time.time()
self.runner.env = self.additional_env
self.runner.startup()
def check_virtual_display(self):
if self.virtual_display:
if not self.virtual_display.is_alive():
self.log.info("Virtual display out: %s", self.virtual_display.stdout)
self.log.warning("Virtual display err: %s", self.virtual_display.stderr)
raise TaurusInternalException("Virtual display failed: %s" % self.virtual_display.return_code)
def check(self):
"""
check if test completed
:return:
"""
if self.widget:
self.widget.update()
self.check_virtual_display()
return self.runner.check()
def report_test_duration(self):
if self.start_time:
self.end_time = time.time()
self.log.debug("Selenium tests ran for %s seconds", self.end_time - self.start_time)
def shutdown(self):
"""
shutdown test_runner
:return:
"""
self.runner.shutdown()
self.report_test_duration()
def post_process(self):
if os.path.exists("geckodriver.log"):
self.engine.existing_artifact("geckodriver.log", True)
self.free_virtual_display()
def has_results(self):
if self.reader and self.reader.read_records:
return True
else:
return False
def get_widget(self):
if not self.widget:
self.widget = SeleniumWidget(self.script, self.runner._stdout_file)
return self.widget
def resource_files(self):
resources = []
self.scenario = self.get_scenario()
script = self.scenario.get(Scenario.SCRIPT, None)
if script:
resources.append(script)
resources.extend(self.scenario.get("additional-classpath", []))
resources.extend(self.settings.get("additional-classpath", []))
testng_config = self._get_testng_xml()
if testng_config:
resources.append(testng_config)
return resources
class Hangouts(NavigationHelpers):
"""
Base class that provide all bunch of options.
"""
# TODO: how to show self.display, self.browser and self.hangout_id in
# docs?
def __init__(self, browser="chrome", executable_path=None):
"""
Initialization does two things:
1. Makes sure that there is active X session.
2. Starts browser.
On initialization it stats X session if can't find 'DISPLAY' in
os.environ. For this purposes used *pyvirtualdisplay* package.
For handling browser used seleniumwrapper library.
"""
# lets start display in case if no is available
self.hangout_id = None
self.display = None
if not os.environ.get('DISPLAY'):
self.display = SmartDisplay()
self.display.start()
kwargs = {}
if browser == "chrome":
kwargs['executable_path'] = executable_path or CHROMEDRV_PATH
self.browser = selwrap.create(browser, **kwargs)
self.video = VideoSettings(self)
self.microphone = MicrophoneSettings(self)
self.audio = AudioSettings(self)
self.bandwidth = BandwidthSettings(self)
def start(self, onair=False):
"""
Start new hangout.
"""
if not self.browser.current_url.startswith(URLS.hangouts_active_list):
self.browser.get(URLS.hangouts_active_list)
self.browser.by_class('opd').click(timeout=0.5)
# G+ opens new window for new hangout, so we need to switch selenium to
# it
# waiting until new window appears
while len(self.browser.window_handles) <= 1:
sleep(0.2) # XXX: add waiting for second window to open
self.browser.close() # closing old window
# TODO: 'Google+' title
self.browser.switch_to_window(self.browser.window_handles[0])
self.click_cancel_button_if_there_is_one(timeout=30)
# setting hangout id property
self.hangout_id = self.browser.current_url.replace(
URLS.hangout_session_base, '', 1).split('?', 1)[0]
def connect(self, hangout_id):
"""
Connect to an existing hangout.
Takes id of targeted hangout as argument
"""
self.hangout_id = hangout_id
self.browser.get(URLS.hangout_session_base + hangout_id)
# there may be a big delay before 'Join' button appears, so we need
# to add longer timeout for this
self.browser.by_text('Join', timeout=60).click(timeout=0.5)
def login(self, username=None, password=None, otp=None):
"""
Log into google plus account.
*opt* argument is one time password and is optional,
set it only if you're 2-factor authorization
"""
# Open login form and sing in with credentials
self.browser.get(URLS.service_login)
self.browser.by_id('Email').send_keys(username)
self.browser.by_id('Passwd').send_keys(password)
self.browser.by_id('signIn').click(timeout=0.5)
# filling up one time password if provides
if otp:
self.browser.by_id('smsUserPin').send_keys(otp)
self.browser.by_id('smsVerifyPin').click(timeout=0.5)
# checking if log in was successful
if not self.is_logged_in:
raise LoginError(
'Wasn\'t able to login. Check if credentials are correct'
'and make sure that you have G+ account activated')
def invite(self, participants):
"""
Invite person or circle to hangout
>>> hangout.invite("*****@*****.**")
>>> hangout.invite(["*****@*****.**", "Circle Name A"])
"""
self.click_cancel_button_if_there_is_one()
if not any(isinstance(participants, i) for i in (list, tuple)):
participants = [
participants,
]
# click on Invite People button
self.click_menu_element('//div[@aria-label="Invite People"]')
input = self.browser.xpath(
'//input[@placeholder="+ Add names, circles, or email addresses"]')
input.send_keys("\n".join(participants) + "\n\n")
self.browser.by_text('Invite').click(timeout=0.5)
def participants(self):
"""
Returns list of current participants
>>> hangout.participants()
['John Doe', ...]
"""
xpath = '//div[@aria-label="Video call participants"]/div'
participants = self.browser.xpath(xpath, eager=True)
return [
p.get_attribute('aria-label').split('Open menu for ')[1]
for p in participants
]
def leave_call(self):
"""
Leave hangout. EQ to click on "Leave call" button.
"""
self.click_menu_element('//div[@aria-label="Leave call"]')
def __del__(self):
# leaving the call first
self.browser.silent = True
try:
self.leave_call()
except:
pass
try:
self.browser.quit()
finally:
if self.display is not None:
self.display.stop()
class SeleniumExecutor(ScenarioExecutor, WidgetProvider, FileLister):
"""
Selenium executor
:type virtual_display: Display
:type runner: AbstractTestRunner
"""
SELENIUM_DOWNLOAD_LINK = "http://selenium-release.storage.googleapis.com/{version}/" \
"selenium-server-standalone-{version}.0.jar"
SELENIUM_VERSION = "2.53"
JUNIT_DOWNLOAD_LINK = "http://search.maven.org/remotecontent?filepath=junit/junit/{version}/junit-{version}.jar"
JUNIT_VERSION = "4.12"
JUNIT_MIRRORS_SOURCE = "http://search.maven.org/solrsearch/select?q=g%3A%22junit%22%20AND%20a%3A%22junit%22%20" \
"AND%20v%3A%22{version}%22&rows=20&wt=json".format(version=JUNIT_VERSION)
HAMCREST_DOWNLOAD_LINK = "https://hamcrest.googlecode.com/files/hamcrest-core-1.3.jar"
SUPPORTED_TYPES = [".py", ".jar", ".java"]
SHARED_VIRTUAL_DISPLAY = {}
def __init__(self):
super(SeleniumExecutor, self).__init__()
self.additional_env = {}
self.virtual_display = None
self.start_time = None
self.end_time = None
self.runner = None
self.widget = None
self.reader = None
self.kpi_file = None
self.err_jtl = None
self.runner_working_dir = None
self.scenario = None
def set_virtual_display(self):
display_conf = self.settings.get("virtual-display")
if display_conf:
if is_windows():
self.log.warning("Cannot have virtual display on Windows, ignoring")
else:
if self.engine in SeleniumExecutor.SHARED_VIRTUAL_DISPLAY:
self.virtual_display = SeleniumExecutor.SHARED_VIRTUAL_DISPLAY[self.engine]
else:
width = display_conf.get("width", 1024)
height = display_conf.get("height", 768)
self.virtual_display = Display(size=(width, height))
msg = "Starting virtual display[%s]: %s"
self.log.info(msg, self.virtual_display.size, self.virtual_display.new_display_var)
self.virtual_display.start()
SeleniumExecutor.SHARED_VIRTUAL_DISPLAY[self.engine] = self.virtual_display
def free_virtual_display(self):
if self.virtual_display and self.virtual_display.is_alive():
self.virtual_display.stop()
if self.engine in SeleniumExecutor.SHARED_VIRTUAL_DISPLAY:
del SeleniumExecutor.SHARED_VIRTUAL_DISPLAY[self.engine]
def _get_script_path(self):
script = self.scenario.get(Scenario.SCRIPT)
if not script:
return None
return self.engine.find_file(script)
def prepare(self):
self.set_virtual_display()
self.scenario = self.get_scenario()
self._verify_script()
self.kpi_file = self.engine.create_artifact("selenium_tests_report", ".csv")
self.err_jtl = self.engine.create_artifact("selenium_tests_err", ".xml")
script_path = self._get_script_path()
script_type = self.detect_script_type(script_path)
runner_config = BetterDict()
if script_type == ".py":
runner_class = NoseTester
runner_config.merge(self.settings.get("selenium-tools").get("nose"))
else: # script_type == ".jar" or script_type == ".java":
runner_class = JUnitTester
runner_config.merge(self.settings.get("selenium-tools").get("junit"))
runner_config['props-file'] = self.engine.create_artifact("customrunner", ".properties")
runner_config["script-type"] = script_type
if self.runner_working_dir is None:
self.runner_working_dir = self.engine.create_artifact(runner_config.get("working-dir", "classes"), "")
runner_config["working-dir"] = self.runner_working_dir
runner_config.get("artifacts-dir", self.engine.artifacts_dir)
runner_config.get("working-dir", self.runner_working_dir)
runner_config.get("report-file", self.kpi_file)
runner_config.get("err-file", self.err_jtl)
runner_config.get("stdout", self.engine.create_artifact("junit", ".out"))
runner_config.get("stderr", self.engine.create_artifact("junit", ".err"))
self._cp_resource_files(self.runner_working_dir)
self.runner = runner_class(runner_config, self)
self.runner.prepare()
self.reader = JTLReader(self.kpi_file, self.log, self.err_jtl)
if isinstance(self.engine.aggregator, ConsolidatingAggregator):
self.engine.aggregator.add_underling(self.reader)
def _verify_script(self):
if not self.scenario.get(Scenario.SCRIPT):
if self.scenario.get("requests"):
self.scenario[Scenario.SCRIPT] = self.__tests_from_requests()
else:
raise RuntimeError("Nothing to test, no requests were provided in scenario")
def _cp_resource_files(self, runner_working_dir):
"""
:return:
"""
script = self._get_script_path()
if script is not None:
if os.path.isdir(script):
shutil.copytree(script, runner_working_dir)
else:
os.makedirs(runner_working_dir)
shutil.copy2(script, runner_working_dir)
@staticmethod
def detect_script_type(script_path):
"""
checks if script is java or python
if it's folder or single script
:param script_path: str
:return:
"""
if not isinstance(script_path, string_types) and not isinstance(script_path, text_type):
raise ValueError("Nothing to test, no files were provided in scenario")
if not os.path.exists(script_path):
raise ValueError("Script %s doesn't exist" % script_path)
file_types = set()
if os.path.isfile(script_path): # regular file received
file_types.add(os.path.splitext(script_path)[1].lower())
else: # dir received: check files
for walk_rec in os.walk(script_path):
for file_name in walk_rec[2]:
file_types.add(os.path.splitext(file_name)[1].lower())
if '.java' in file_types:
file_ext = '.java'
elif '.py' in file_types:
file_ext = '.py'
elif '.jar' in file_types:
file_ext = '.jar'
else:
raise ValueError("Unsupported script type: %s" % script_path)
return file_ext
def startup(self):
"""
Start runner
:return:
"""
self.start_time = time.time()
self.runner.env = self.additional_env
self.runner.run_tests()
def check(self):
"""
check if test completed
:return:
"""
if self.widget:
self.widget.update()
if self.virtual_display:
if not self.virtual_display.is_alive():
self.log.info("Virtual display out: %s", self.virtual_display.stdout)
self.log.warning("Virtual display err: %s", self.virtual_display.stderr)
raise RuntimeError("Virtual display failed: %s" % self.virtual_display.return_code)
return self.runner.is_finished()
def shutdown(self):
"""
shutdown test_runner
:return:
"""
self.runner.shutdown()
if self.start_time:
self.end_time = time.time()
self.log.debug("Selenium tests ran for %s seconds", self.end_time - self.start_time)
def post_process(self):
self.free_virtual_display()
if self.reader and not self.reader.read_records:
raise RuntimeWarning("Empty results, most likely Selenium failed")
def get_widget(self):
if not self.widget:
self.widget = SeleniumWidget(self.scenario.get(Scenario.SCRIPT), self.runner.settings.get("stdout"))
return self.widget
def resource_files(self):
if not self.scenario:
self.scenario = self.get_scenario()
if Scenario.SCRIPT not in self.scenario:
return []
script_path = self._get_script_path()
if os.path.isdir(script_path):
files = next(os.walk(script_path))
resources = [os.path.join(files[0], f) for f in files[2]]
else:
resources = [script_path]
return resources
def __tests_from_requests(self):
filename = self.engine.create_artifact("test_requests", ".py")
nose_test = SeleniumScriptBuilder(self.scenario, self.log)
if self.virtual_display:
nose_test.window_size = self.virtual_display.size
nose_test.gen_test_case()
nose_test.save(filename)
return filename
class SeleniumExecutor(ScenarioExecutor, WidgetProvider, FileLister):
"""
Selenium executor
:type virtual_display: Display
:type runner: AbstractTestRunner
"""
SELENIUM_DOWNLOAD_LINK = "http://selenium-release.storage.googleapis.com/{version}/" \
"selenium-server-standalone-{version}.0.jar"
SELENIUM_VERSION = "2.53"
JUNIT_DOWNLOAD_LINK = "http://search.maven.org/remotecontent?filepath=junit/junit/" \
"{version}/junit-{version}.jar"
JUNIT_VERSION = "4.12"
JUNIT_MIRRORS_SOURCE = "http://search.maven.org/solrsearch/select?q=g%3A%22junit%22%20AND%20a%3A%22" \
"junit%22%20AND%20v%3A%22{version}%22&rows=20&wt=json".format(version=JUNIT_VERSION)
HAMCREST_DOWNLOAD_LINK = "http://search.maven.org/remotecontent?filepath=org/hamcrest/hamcrest-core" \
"/1.3/hamcrest-core-1.3.jar"
JSON_JAR_DOWNLOAD_LINK = "http://search.maven.org/remotecontent?filepath=org/json/json/20160810/json-20160810.jar"
SUPPORTED_TYPES = ["python-nose", "java-junit", "ruby-rspec"]
SHARED_VIRTUAL_DISPLAY = {}
def __init__(self):
super(SeleniumExecutor, self).__init__()
self.additional_env = {}
self.virtual_display = None
self.end_time = None
self.runner = None
self.report_file = None
self.scenario = None
self.script = None
self.self_generated_script = False
self.generated_methods = BetterDict()
self.runner_working_dir = None
def set_virtual_display(self):
display_conf = self.settings.get("virtual-display")
if display_conf:
if is_windows():
self.log.warning("Cannot have virtual display on Windows, ignoring")
else:
if self.engine in SeleniumExecutor.SHARED_VIRTUAL_DISPLAY:
self.virtual_display = SeleniumExecutor.SHARED_VIRTUAL_DISPLAY[self.engine]
else:
width = display_conf.get("width", 1024)
height = display_conf.get("height", 768)
self.virtual_display = Display(size=(width, height))
msg = "Starting virtual display[%s]: %s"
self.log.info(msg, self.virtual_display.size, self.virtual_display.new_display_var)
self.virtual_display.start()
SeleniumExecutor.SHARED_VIRTUAL_DISPLAY[self.engine] = self.virtual_display
def free_virtual_display(self):
if self.virtual_display and self.virtual_display.is_alive():
self.virtual_display.stop()
if self.engine in SeleniumExecutor.SHARED_VIRTUAL_DISPLAY:
del SeleniumExecutor.SHARED_VIRTUAL_DISPLAY[self.engine]
def get_script_path(self, scenario=None):
if scenario:
return super(SeleniumExecutor, self).get_script_path(scenario)
else:
return self.engine.find_file(self.script)
def get_runner_working_dir(self):
if self.runner_working_dir is None:
self.runner_working_dir = self.engine.create_artifact("classes", "")
return self.runner_working_dir
def _create_runner(self, report_file):
script_path = self.get_script_path()
script_type = self.detect_script_type(script_path)
runner_config = BetterDict()
if script_type == "python-nose":
runner_class = NoseTester
runner_config.merge(self.settings.get("selenium-tools").get("nose"))
elif script_type == "java-junit":
runner_class = JUnitTester
runner_config.merge(self.settings.get("selenium-tools").get("junit"))
runner_config['working-dir'] = self.get_runner_working_dir()
runner_config['props-file'] = self.engine.create_artifact("customrunner", ".properties")
elif script_type == "ruby-rspec":
runner_class = RSpecTester
elif script_type == "js-mocha":
runner_class = MochaTester
else:
raise ValueError("Unsupported script type: %s" % script_type)
runner_config["script"] = script_path
runner_config["script-type"] = script_type
runner_config["artifacts-dir"] = self.engine.artifacts_dir
runner_config["report-file"] = report_file
runner_config["stdout"] = self.engine.create_artifact("selenium", ".out")
runner_config["stderr"] = self.engine.create_artifact("selenium", ".err")
return runner_class(runner_config, self)
def _register_reader(self, report_file):
if self.engine.is_functional_mode():
reader = FuncSamplesReader(report_file, self.log, self.generated_methods)
if isinstance(self.engine.aggregator, FunctionalAggregator):
self.engine.aggregator.add_underling(reader)
else:
reader = LoadSamplesReader(report_file, self.log, self.generated_methods)
if isinstance(self.engine.aggregator, ConsolidatingAggregator):
self.engine.aggregator.add_underling(reader)
return reader
def prepare(self):
self.set_virtual_display()
self.scenario = self.get_scenario()
self.__setup_script()
self.report_file = self.engine.create_artifact("selenium_tests_report", ".ldjson")
self.runner = self._create_runner(self.report_file)
self.runner.prepare()
self.reader = self._register_reader(self.report_file)
def __setup_script(self):
if Scenario.SCRIPT in self.scenario and self.scenario.get(Scenario.SCRIPT):
self.script = self.scenario.get(Scenario.SCRIPT)
elif "requests" in self.scenario:
self.script = self.__tests_from_requests()
self.self_generated_script = True
else:
raise ValueError("Nothing to test, no requests were provided in scenario")
def detect_script_type(self, script_path):
if not isinstance(script_path, string_types) and not isinstance(script_path, text_type):
raise ValueError("Nothing to test, no files were provided in scenario")
if not os.path.exists(script_path):
raise ValueError("Script %s doesn't exist" % script_path)
if "language" in self.execution:
lang = self.execution["language"]
if lang not in self.SUPPORTED_TYPES:
tmpl = "Language '%s' is not supported. Supported languages are: %s"
raise ValueError(tmpl % (lang, self.SUPPORTED_TYPES))
self.log.debug("Using script type: %s", lang)
return lang
file_types = set()
if os.path.isfile(script_path): # regular file received
file_types.add(os.path.splitext(script_path)[1].lower())
else: # dir received: check contained files
for file_name in get_files_recursive(script_path):
file_types.add(os.path.splitext(file_name)[1].lower())
if '.java' in file_types or '.jar' in file_types:
script_type = 'java-junit'
elif '.py' in file_types:
script_type = 'python-nose'
elif '.rb' in file_types:
script_type = 'ruby-rspec'
elif '.js' in file_types:
script_type = 'js-mocha'
else:
raise ValueError("Unsupported script type: %s" % script_path)
self.log.debug("Detected script type: %s", script_type)
return script_type
def startup(self):
"""
Start runner
:return:
"""
self.start_time = time.time()
self.runner.env = self.additional_env
self.runner.run_tests()
def check_virtual_display(self):
if self.virtual_display:
if not self.virtual_display.is_alive():
self.log.info("Virtual display out: %s", self.virtual_display.stdout)
self.log.warning("Virtual display err: %s", self.virtual_display.stderr)
raise RuntimeError("Virtual display failed: %s" % self.virtual_display.return_code)
def check(self):
"""
check if test completed
:return:
"""
if self.widget:
self.widget.update()
self.check_virtual_display()
return self.runner.is_finished()
def report_test_duration(self):
if self.start_time:
self.end_time = time.time()
self.log.debug("Selenium tests ran for %s seconds", self.end_time - self.start_time)
def shutdown(self):
"""
shutdown test_runner
:return:
"""
self.runner.shutdown()
self.report_test_duration()
def post_process(self):
self.free_virtual_display()
def has_results(self):
if self.reader and self.reader.read_records:
return True
else:
return False
def get_widget(self):
if not self.widget:
self.widget = SeleniumWidget(self.script, self.runner.settings.get("stdout"))
return self.widget
def resource_files(self):
self.scenario = self.get_scenario()
self.__setup_script()
script_path = self.get_script_path()
resources = []
if script_path is not None:
resources.append(script_path)
return resources
def __tests_from_requests(self):
filename = self.engine.create_artifact("test_requests", ".py")
wdlog = self.engine.create_artifact('webdriver', '.log')
nose_test = SeleniumScriptBuilder(self.scenario, self.log, wdlog)
if self.virtual_display:
nose_test.window_size = self.virtual_display.size
self.generated_methods.merge(nose_test.build_source_code())
nose_test.save(filename)
return filename
class Browser:
def __init__(self,
cookies_file='data/cookies.json',
virtual_display=True,
log_file=None):
self.logger = logging.getLogger()
self.logger.setLevel(logging.INFO)
if log_file:
fh = logging.FileHandler(log_file)
fh.setLevel(logging.INFO)
self.logger.addHandler(fh)
self.cookies_file = cookies_file
if virtual_display:
self.display = SmartDisplay(visible=0,
backend='xvfb',
size=(1920, 1080))
self.display.start()
else:
self.display = None
self._reload_driver()
def quit(self):
if hasattr(self, 'driver'):
self.clear_cookies()
self.driver.quit()
def save_cookies(self):
self.logger.info('Saving cookies')
cookies = self.driver.get_cookies()
with open(self.cookies_file, 'w') as f:
json.dump(cookies, f)
def get_cookies(self):
return self.driver.get_cookies()
def load_cookies(self, path=None):
# Note: to load cookies for a domain, you must
# first navigate to that domain.
self.logger.info('Loading cookies')
if path is None:
path = self.cookies_file
if path.endswith('.json'):
try:
with open(self.cookies_file, 'r') as f:
cookies = json.load(f)
except FileNotFoundError:
return False
for c in cookies:
if isinstance(c.get('expiry'), float):
c['expiry'] = int(c['expiry'])
self.driver.add_cookie(c)
elif path.endswith('.txt'):
# Assume Mozilla/Netscape format
jar = MozillaCookieJar(path)
jar.load()
for c in jar:
cookie = {
k: getattr(c, k)
for k in ['domain', 'name', 'value', 'secure', 'path']
}
if c.expires: cookie['expiry'] = c.expires
self.driver.add_cookie(cookie)
else:
raise Exception('Unrecognized cookie extension')
return True
def clear_cookies(self):
self.driver.delete_all_cookies()
def _reload_driver(self, opts=[]):
self.quit()
options = webdriver.ChromeOptions()
for opt in opts:
options.add_argument(opt)
# Necessary for headless on server
options.add_argument('--no-sandbox')
options.add_experimental_option('excludeSwitches',
['enable-automation'])
options.add_experimental_option('useAutomationExtension', False)
# Don't ask to save passwords
options.add_experimental_option(
'prefs', {
'credentials_enable_service': False,
'profile.password_manager_enabled': False
})
self.driver = webdriver.Chrome(chrome_options=options)
self.driver.maximize_window()
self.wait = WebDriverWait(self.driver, 5)
self.actions = ActionChains(self.driver)
@property
def user_agent(self):
return self.driver.execute_script('return navigator.userAgent')
def screenshot(self, fname=None):
fname = fname or 'data/shots/{}.png'.format(
datetime.utcnow().timestamp())
# if self.display:
# img = self.display.waitgrab()
# img.save(fname)
self.driver.save_screenshot(fname)
return fname
def visit(self, url):
self.driver.get(url)
def input(self, selector, value, wait=True):
if wait: self.wait.until(visible((By.CSS_SELECTOR, selector)))
self.driver.find_element_by_css_selector(selector).send_keys(value)
def click(self, selector, wait=True):
if wait: self.wait.until(clickable((By.CSS_SELECTOR, selector)))
self.driver.find_element_by_css_selector(selector).click()
def select(self, selector, value):
# Dropdowns
select = Select(self.driver.find_element_by_css_selector(selector))
select.select_by_value(value)
def html(self):
html = self.driver.find_element_by_tag_name('html').get_attribute(
'innerHTML')
return lxml.html.fromstring(html)
def wait_for(self, selector):
self.wait.until(visible((By.CSS_SELECTOR, selector)))
def execute_script(self, script):
self.driver.execute_script(script)
@property
def url(self):
return self.driver.current_url
class Hangouts(NavigationHelpers):
"""
Base class that provide all bunch of options.
"""
# TODO: how to show self.display, self.browser and self.hangout_id in
# docs?
def __init__(self, browser="chrome", executable_path=None):
"""
Initialization does two things:
1. Makes sure that there is active X session.
2. Starts browser.
On initialization it stats X session if can't find 'DISPLAY' in
os.environ. For this purposes used *pyvirtualdisplay* package.
For handling browser used seleniumwrapper library.
"""
# lets start display in case if no is available
self.hangout_id = None
self.display = None
if not os.environ.get('DISPLAY'):
self.display = SmartDisplay()
self.display.start()
kwargs = {}
if browser == "chrome":
kwargs['executable_path'] = executable_path or CHROMEDRV_PATH
self.browser = selwrap.create(browser, **kwargs)
self.video = VideoSettings(self)
self.microphone = MicrophoneSettings(self)
self.audio = AudioSettings(self)
self.bandwidth = BandwidthSettings(self)
def start(self, onair=False):
"""
Start new hangout.
"""
if not self.browser.current_url.startswith(URLS.hangouts_active_list):
self.browser.get(URLS.hangouts_active_list)
self.browser.by_class('opd').click(timeout=0.5)
# G+ opens new window for new hangout, so we need to switch selenium to
# it
# waiting until new window appears
while len(self.browser.window_handles) <= 1:
sleep(0.2) # XXX: add waiting for second window to open
self.browser.close() # closing old window
# TODO: 'Google+' title
self.browser.switch_to_window(self.browser.window_handles[0])
self.click_cancel_button_if_there_is_one(timeout=30)
# setting hangout id property
self.hangout_id = self.browser.current_url.replace(
URLS.hangout_session_base, '', 1).split('?', 1)[0]
def connect(self, hangout_id):
"""
Connect to an existing hangout.
Takes id of targeted hangout as argument
"""
self.hangout_id = hangout_id
self.browser.get(URLS.hangout_session_base + hangout_id)
# there may be a big delay before 'Join' button appears, so we need
# to add longer timeout for this
self.browser.by_text('Join', timeout=60).click(timeout=0.5)
def login(self, username=None, password=None, otp=None):
"""
Log into google plus account.
*opt* argument is one time password and is optional,
set it only if you're 2-factor authorization
"""
# Open login form and sing in with credentials
self.browser.get(URLS.service_login)
self.browser.by_id('Email').send_keys(username)
self.browser.by_id('Passwd').send_keys(password)
self.browser.by_id('signIn').click(timeout=0.5)
# filling up one time password if provides
if otp:
self.browser.by_id('smsUserPin').send_keys(otp)
self.browser.by_id('smsVerifyPin').click(timeout=0.5)
# checking if log in was successful
if not self.is_logged_in:
raise LoginError(
'Wasn\'t able to login. Check if credentials are correct'
'and make sure that you have G+ account activated')
def invite(self, participants):
"""
Invite person or circle to hangout
>>> hangout.invite("*****@*****.**")
>>> hangout.invite(["*****@*****.**", "Circle Name A"])
"""
self.click_cancel_button_if_there_is_one()
if not any(isinstance(participants, i) for i in (list, tuple)):
participants = [participants, ]
# click on Invite People button
self.click_menu_element('//div[@aria-label="Invite People"]')
input = self.browser.xpath(
'//input[@placeholder="+ Add names, circles, or email addresses"]')
input.send_keys("\n".join(participants) + "\n\n")
self.browser.by_text('Invite').click(timeout=0.5)
def participants(self):
"""
Returns list of current participants
>>> hangout.participants()
['John Doe', ...]
"""
xpath = '//div[@aria-label="Video call participants"]/div'
participants = self.browser.xpath(xpath, eager=True)
return [p.get_attribute('aria-label').split('Open menu for ')[1]
for p in participants]
def leave_call(self):
"""
Leave hangout. EQ to click on "Leave call" button.
"""
self.click_menu_element('//div[@aria-label="Leave call"]')
def __del__(self):
# leaving the call first
self.browser.silent = True
try:
self.leave_call()
except:
pass
try:
self.browser.quit()
finally:
if self.display is not None:
self.display.stop()
