def get_display(self):
"""Returns the first display available"""
# Check if an Xorg server is listening
# import subprocess as sp
# if not hasattr(sp, 'DEVNULL'):
# setattr(sp, 'DEVNULL', os.devnull)
# x_listening = bool(sp.call('ps au | grep -v grep | grep -i xorg',
# shell=True, stdout=sp.DEVNULL))
if self._display is not None:
return ":%d" % self._display.new_display
sysdisplay = None
if self._config.has_option("execution", "display_variable"):
sysdisplay = self._config.get("execution", "display_variable")
sysdisplay = sysdisplay or os.getenv("DISPLAY")
if sysdisplay:
from collections import namedtuple
def _mock():
pass
# Store a fake Xvfb object. Format - <host>:<display>[.<screen>]
ndisp = sysdisplay.split(":")[-1].split(".")[0]
Xvfb = namedtuple("Xvfb", ["new_display", "stop"])
self._display = Xvfb(int(ndisp), _mock)
return self.get_display()
else:
if "darwin" in sys.platform:
raise RuntimeError(
"Xvfb requires root permissions to run in OSX. Please "
"make sure that an X server is listening and set the "
"appropriate config on either $DISPLAY or nipype's "
'"display_variable" config. Valid X servers include '
"VNC, XQuartz, or manually started Xvfb.")
# If $DISPLAY is empty, it confuses Xvfb so unset
if sysdisplay == "":
del os.environ["DISPLAY"]
try:
from xvfbwrapper import Xvfb
except ImportError:
raise RuntimeError(
"A display server was required, but $DISPLAY is not "
"defined and Xvfb could not be imported.")
self._display = Xvfb(nolisten="tcp")
self._display.start()
# Older versions of xvfbwrapper used vdisplay_num
if not hasattr(self._display, "new_display"):
setattr(self._display, "new_display",
self._display.vdisplay_num)
return self.get_display()
def get_display(self):
"""Returns the first display available"""
# Check if an Xorg server is listening
# import subprocess as sp
# if not hasattr(sp, 'DEVNULL'):
# setattr(sp, 'DEVNULL', os.devnull)
# x_listening = bool(sp.call('ps au | grep -v grep | grep -i xorg',
# shell=True, stdout=sp.DEVNULL))
if self._display is not None:
return ':%d' % self._display.new_display
sysdisplay = None
if self._config.has_option('execution', 'display_variable'):
sysdisplay = self._config.get('execution', 'display_variable')
sysdisplay = sysdisplay or os.getenv('DISPLAY')
if sysdisplay:
from collections import namedtuple
def _mock():
pass
# Store a fake Xvfb object. Format - <host>:<display>[.<screen>]
ndisp = sysdisplay.split(':')[-1].split('.')[0]
Xvfb = namedtuple('Xvfb', ['new_display', 'stop'])
self._display = Xvfb(int(ndisp), _mock)
return self.get_display()
else:
if 'darwin' in sys.platform:
raise RuntimeError(
'Xvfb requires root permissions to run in OSX. Please '
'make sure that an X server is listening and set the '
'appropriate config on either $DISPLAY or nipype\'s '
'"display_variable" config. Valid X servers include '
'VNC, XQuartz, or manually started Xvfb.')
# If $DISPLAY is empty, it confuses Xvfb so unset
if sysdisplay == '':
del os.environ['DISPLAY']
try:
from xvfbwrapper import Xvfb
except ImportError:
raise RuntimeError(
'A display server was required, but $DISPLAY is not '
'defined and Xvfb could not be imported.')
self._display = Xvfb(nolisten='tcp')
self._display.start()
# Older versions of xvfbwrapper used vdisplay_num
if not hasattr(self._display, 'new_display'):
setattr(self._display, 'new_display',
self._display.vdisplay_num)
return self.get_display()
def makeHeadLess(self, config):
if config["headless"]:
self.xvfb = Xvfb(width=1280, height=720)
self.xvfb.start()
return config["headless"]
def start_xvfb(module):
try:
xvfb = Xvfb(width=1280, height=720)
xvfb.start()
atexit.register(xvfb.stop)
except:
module.fail_json(msg="xvfb broke")
def tree_builder(node_file, function_folder):
"""
Perform tree builder script to extract csv, tsv and svg.
To correctly run tree_builder needs Xorg. We are using xvfb-run to workaround
on "x-server-less" systems.
Script usage: python new_tree_builder_for_perl_tango.py -d visualization_ITSoneDB_r131.dmp
"""
sys.stdout.write('\n### Step 5: Tree Builder\n')
# xvbf-run issue:
# /bin/xvfb-run: line 181: 17066 Segmentation fault DISPLAY=:$SERVERNUM XAUTHORITY=$AUTHFILE "$@" 2>&1
# solution: https://unix.stackexchange.com/questions/152957/xvfb-run-aborting-on-cluster
###command = 'xvfb-run -d -e /tmp/biomas_xvfb.log '+__python_executable__+' '+__location__+'/new_tree_builder_for_perl_tango.py -d '+ visualization
###command = 'xvfb-run '+__python_executable__+' '+__location__+'/new_tree_builder_for_perl_tango.py -d '+ visualization
#stdout, stderr, status = run_command(command)
command = __python_executable__+' '+__location__+'/new_tree_builder_for_perl_tango.py -d '+ node_file + ' -F ' + function_folder
sys.stdout.write(command)
from xvfbwrapper import Xvfb
vdisplay = Xvfb()
vdisplay.start()
stdout, stderr, status = run_command(command)
vdisplay.stop()
# Read stdout for errors
if status == 0:
sys.stdout.write(stdout)
else:
sys.stderr.write(stderr)
def handle(self, *args, **options):
xvfb = Xvfb(width=1600, height=720)
xvfb.start()
browser = WebDriver()
browser.get(settings.DOMAIN)
sleep(1)
browser.find_element_by_css_selector("#disclaimer button").click()
sleep(0.7)
browser.find_elements_by_css_selector(".officer .checkmark")[4].click()
sleep(2)
browser.find_element_by_css_selector(
".complaint-row .col-md-3").click()
sleep(1)
content = browser.get_screenshot_as_png()
now = datetime.datetime.now().strftime("%Y-%m-%d %H-%M-%S")
file_name = "{now}.png".format(now=now)
file_path = os.path.join(settings.BASE_DIR, 'static', file_name)
with open(file_path, "wb") as f:
f.write(content)
browser.quit()
email = EmailMessage(subject='CPDB Homepage screenshot %s' % now,
body='FYI',
to=['*****@*****.**'])
email.attach_file(file_path)
email.send()
xvfb.stop()
def setUpClass(cls):
# Avoid problems with the overlaping menu when clicking
settings.ADMIN_TOOLS_MENU = 'admin_tools.menu.Menu'
cls.vdisplay = Xvfb()
cls.vdisplay.start()
cls.selenium = WebDriver()
super(BaseLiveServerTestCase, cls).setUpClass()
def setUpClass(cls):
if os.getenv('TRAVIS', False):
from xvfbwrapper import Xvfb # virtual X server for plots under headless LINUX travis testing is needed
cls.vdisplay = Xvfb()
cls.vdisplay.start()
cls.output_folder = tests_data_folder
cls.pixel_size = (250, 50) # in um
def start_xvfb():
from xvfbwrapper import Xvfb
if "DISPLAY" in os.environ:
del os.environ["DISPLAY"]
xvfb = Xvfb()
xvfb.start()
atexit.register(xvfb.stop)
def work():
while True:
try:
clone = requests.get("{}/api/clone".format(url)).json()
print(clone)
vdisplay = Xvfb()
vdisplay.start()
display = Display(visible=0, size=(800, 600))
display.start()
driver = helper._init(clone['ip'], clone['port'], clone['c_user'],
clone['xs'])
try:
helper.newest_message(driver)
helper.request_message(driver)
#active_user_links = helper.active_conversations(driver)
#helper.message_to_active_users(driver, active_user_links)
except Exception as e:
print("Ex 1 : {}".format(e))
driver.save_screenshot('log-{}.{}'.format(clone['c_user'], 'png'))
driver.quit()
display.stop()
vdisplay.stop()
except Exception as e:
print(e)
time.sleep(5)
def test_start_with_arbitrary_args_str(self):
xvfb = Xvfb(extra_args_str='+extension GLX -noreset')
self.addCleanup(xvfb.stop)
xvfb.start()
display_var = ':{}'.format(xvfb.new_display)
self.assertEqual(display_var, os.environ['DISPLAY'])
self.assertIsNotNone(xvfb.proc)
def open_display(self):
"""Display of the gui page after connecting through some random port
"""
from pyvirtualdisplay.randomize import Randomizer
from boardfarm import config
if config.default_headless:
self.display = None
return
if config.default_display_backend == 'xvnc':
xc, yc = config.default_display_backend_size.split('x')
x = int(xc)
y = int(yc)
r = Randomizer(
) if config.default_display_backend_port == 0 else None
self.display = Display(backend=config.default_display_backend,
rfbport=config.default_display_backend_port,
rfbauth=os.environ['HOME'] + '/.vnc/passwd',
visible=0,
randomizer=r,
size=(x, y))
elif config.default_display_backend == 'xvfb':
self.display = Xvfb()
else:
raise Exception("backend not yet tested!")
self.display.start()
def worker(game, play_game, solution_q, reward_q, stop, show_screen):
""" A worker that plays the game with a proposed solution.
Workers fetch proposed solutions from solution_q (Solution Queue) and put the
matching reward into the reward_q (Reward Queue). If a stop signal is received the
worker terminates. Rewards are matched to solutions at the receiving and by using
the solution_id as an identifier.
"""
if not show_screen:
vdisplay = Xvfb()
vdisplay.start()
while not stop.is_set():
try:
(solution_id, solution) = solution_q.get(False, 0.1)
except queue.Empty:
pass
else:
name = multiprocessing.current_process().name
logger.info("%s started working on %s", name, solution_id)
reward = play_game(solution=solution)
reward_q.put((solution_id, reward))
if not show_screen:
vdisplay.stop()
def __init__(self):
try:
self.display = Xvfb()
self.display.start()
except ErrorReturnCode:
msgerr("Failed start Xvfb server")
raise
def plot_response(response_src, out_png=False):
# start virtual display
print("starting Xvfb");
vdisplay = Xvfb()
vdisplay.start()
response_src = np.loadtxt(src_txt)
if len(response_src.shape) > 1:
response_src = response_src[1]
sphere = get_sphere('symmetric724')
sh_resp = AxSymShResponse(0, response_src)
sig_resp = sh_resp.on_sphere(sphere)
sig_resp = sig_resp[None, None, None, :]
ren = window.Renderer()
sphere_actor = actor.odf_slicer(sig_resp, sphere=sphere,colormap='blues')
ren.add(sphere_actor)
my_camera = ren.camera()
my_camera.SetPosition(1.62, -9.19, 4.01)
my_camera.SetFocalPoint(0.01, -0.46, -0.19)
my_camera.SetViewUp(0.24, 0.46, 0.86)
if out_png != False:
window.record(ren, out_path=out_png, magnification=10, size=(60, 60))
else:
window.show(ren, reset_camera=False)
print('Camera Settings')
print('Position: ', '(%.2f, %.2f, %.2f)' % my_camera.GetPosition())
print('Focal Point: ', '(%.2f, %.2f, %.2f)' % my_camera.GetFocalPoint())
print('View Up: ', '(%.2f, %.2f, %.2f)' % my_camera.GetViewUp())
vdisplay.stop()
def get_page(url):
path_to_chromedriver = '/usr/bin/chromedriver' # change path as needed
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--no-sandbox')
vdisplay = Xvfb()
vdisplay.start()
display = Display(visible=0, size=(1024, 768))
display.start()
browser = webdriver.Chrome(
executable_path=path_to_chromedriver,
service_args=["--verbose", "--log-path=/tmp/CHROMIUM_LOG"],
options=chrome_options)
browser.get(url)
time.sleep(2)
page = browser.page_source
browser.close()
display.stop()
vdisplay.stop()
return page
def get(ui_addr, user, ui_flag):
ui_email = user + '@somemail.com'
ui_passwd = hashlib.sha224(user.encode('utf-8')).hexdigest()
vdisplay = Xvfb()
vdisplay.start()
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--no-sandbox')
browser = webdriver.Chrome(chrome_options=chrome_options)
browser.get('http://{}:3000/login'.format(ui_addr))
browser.set_window_size(1920, 1080)
try:
browser.find_element_by_xpath('//*[@id="email"]').send_keys(ui_email)
browser.find_element_by_xpath('//*[@id="password"]').send_keys(
ui_passwd)
browser.find_element_by_xpath(
'/html/body/div/div/div[2]/form/div[3]/input').click()
browser.find_element_by_xpath('/html/body/div/div[1]/div/a').click()
browser.get_screenshot_as_file('/tmp/get.png')
flag = browser.find_element_by_xpath(
'/html/body/div/div[1]/div[1]/div/p').text
except NoSuchElementException:
return (CORRUPT)
if ('sibirctf:' + ui_flag) == flag:
return (SUCCESS)
else:
return (MUMBLE)
def get_seller_phone(url):
# телефон показывается в виде картинки, используем selenium и pytesseract
vdisplay = Xvfb()
vdisplay.start()
driver = webdriver.Chrome(options=options)
driver.set_window_size(1920, 1080)
driver.get(url)
try:
button = driver.find_element_by_xpath('//a[@class="button item-phone-button js-item-phone-button '
'button-origin button-origin-blue button-origin_full-width '
'button-origin_large-extra item-phone-button_hide-phone '
'item-phone-button_card js-item-phone-button_card"]')
button.click()
time.sleep(2)
driver.save_screenshot("phone_number.png")
image = driver.find_element_by_xpath('//div[@class="item-phone-big-number js-item-phone-big-number"]//*')
cropped = Image.open("phone_number.png")
x, y = image.location["x"], image.location["y"]
width, height = image.size["width"], image.size["height"]
cropped.crop((x, y, x + width, y + height)).save("phone.gif")
phone = Image.open("phone.gif")
phone_text = image_to_string(phone)
except Exception as e:
with open("logs.txt", "a", encoding="utf8") as file:
file.write(str(e) + " avito get_seller_phone\n")
phone_text = "Не указано"
driver.quit()
vdisplay.stop()
return phone_text
def start_virtual_display(self,
width=1440,
height=900,
color_depth=24,
**kwargs):
"""Starts virtual display which will be
destroyed after test execution will be end
*Arguments:*
- width: a width to be set in pixels
- height: a height to be set in pixels
- color_depth: a color depth to be used
- kwargs: extra parameters
*Example:*
| Start Virtual Display |
| Start Virtual Display | 1920 | 1080 |
| Start Virtual Display | ${1920} | ${1080} | ${16} |
"""
if self._display is None:
logger.info("Using virtual display: '{0}x{1}x{2}'".format(
width, height, color_depth))
self._display = Xvfb(int(width), int(height), int(color_depth),
**kwargs)
self._display.start()
atexit.register(self._display.stop)
def __init__(self, width, height, colordepth):
self.width = width
self.height = height
self.xvfb = Xvfb(width=width, height=height, colordepth=colordepth)
self.xvfb.start()
self.fp = webdriver.FirefoxProfile()
self.fp.accept_untrusted_certs = True
self.fp.set_preference('intl.accept_languages', 'ja-JP, ja')
self.fp.set_preference('browser.cache.disk.enable', False)
self.fp.set_preference('browser.cache.disk_cache_ssl', False)
self.fp.set_preference('browser.cache.memory.enable', False)
self.fp.set_preference('browser.cache.offline.enable', False)
self.fp.set_preference('browser.safebrowsing.enabled', False)
self.fp.set_preference('browser.safebrowsing.malware.enabled', False)
self.fp.set_preference('browser.safebrowsing.downloads.enabled', False)
self.fp.set_preference('services.sync.prefs.sync.browser.safebrowsing.enabled', False)
self.fp.set_preference('services.sync.prefs.sync.browser.safebrowsing.malware.enabled', False)
self.fp.set_preference('browser.safebrowsing.provider.mozilla.gethashURL', '')
self.fp.set_preference('browser.safebrowsing.provider.mozilla.updateURL', '')
self.fp.set_preference('browser.search.geoip.url', '')
self.fp.set_preference('geo.enabled', False)
self.fp.set_preference('app.update.enabled', False)
self.fp.set_preference('browser.search.update', False)
self.fp.set_preference('network.http.use-cache', False)
self.fp.set_preference('network.http.pipelining', True)
self.fp.set_preference('network.cookie.enableForCurrentSessionOnly', True)
self.fp.set_preference('browser.chrome.favicons', False)
self.fp.set_preference('browser.chrome.toolbar_tips', False)
self.fp.set_preference('browser.shell.checkDefaultBrowser', False)
self.fp.set_preference('accessibility.blockautorefresh', True)
self.fp.set_preference('security.tls.unrestricted_rc4_fallback', True)
self.fp.set_preference('security.tls.version.max', 3)
self.fp.set_preference('security.tls.version.min', 0)
self.fp.update_preferences()
def tree_builder(visualization):
"""
Perform tree builder script to extract csv, tsv and svg.
To correctly run tree_builder needs Xorg. We are using xvfb-run to workaround
on "x-server-less" systems.
Script usage: python new_tree_builder_for_perl_tango.py -d visualization_ITSoneDB_r131.dmp
"""
sys.stdout.write('\n### Step 5: Tree Builder\n')
#command = 'xvfb-run -a -e /export/xvfb.log '+__python_executable__+' '+__location__+'/new_tree_builder_for_perl_tango.py -d '+ visualization
#stdout, stderr, status = run_command(command)
command = __python_executable__+' '+__location__+'/new_tree_builder_for_perl_tango.py -d '+ visualization
from xvfbwrapper import Xvfb
vdisplay = Xvfb()
vdisplay.start()
stdout, stderr, status = run_command(command)
vdisplay.stop()
# Read stdout for errors
if status == 0:
sys.stdout.write(stdout)
else:
sys.stderr.write(stdout)
sys.stderr.write(stderr)
def test_start(self):
xvfb = Xvfb()
self.addCleanup(xvfb.stop)
xvfb.start()
display_var = ':{}'.format(xvfb.new_display)
self.assertEqual(display_var, os.environ['DISPLAY'])
self.assertIsNotNone(xvfb.proc)
def runner(context):
driverManager = context['driverManager']
user = context['user']
console: Console = context['console']
playlist = context['playlist']
queueUrl = context['queueUrl']
proxy = context['proxy']
shutdownEvent = context['shutdownEvent']
pid = current_process().pid
try:
vdisplay = Xvfb(width=1280,
height=1024,
colordepth=24,
tempdir=None,
noreset='+render')
vdisplay.start()
driverData = driverManager.getDriver(type='chrome',
uid=pid,
user=user,
proxy=proxy)
if not driverData:
return
driver = driverData['driver']
userDataDir = driverData['userDataDir']
if not driver:
return
except:
console.error('Unavailale webdriver: %s' % format_exc())
else:
try:
spotify = Spotify.Adapter(driver, console, shutdownEvent)
if not shutdownEvent.is_set():
if spotify.register(user):
message = {'user': user, 'playlist': playlist}
client = client('sqs')
client.send_message(
QueueUrl=queueUrl,
MessageBody=dumps(message),
DelaySeconds=1,
)
except:
console.exception()
if driver:
try:
driver.quit()
except:
pass
if userDataDir:
try:
rmtree(path=userDataDir, ignore_errors=True)
except:
pass
if vdisplay:
try:
vdisplay.stop()
except:
pass
def test_start_with_arbitrary_kwargs(self):
xvfb = Xvfb(nolisten='tcp')
self.addCleanup(xvfb.stop)
xvfb.start()
display_var = ':{}'.format(xvfb.new_display)
self.assertEqual(display_var, os.environ['DISPLAY'])
self.assertIsNotNone(xvfb.proc)
def jupyter_play(env, agent, steps=1000):
from IPython import display
policy = agent.get_policy()
vdisplay = Xvfb(width=1280, height=740)
vdisplay.start()
observation = env.reset()
def showarray(a, fmt='png'):
a = np.uint8(a)
f = io.BytesIO()
ima = PIL.Image.fromarray(a).save(f, fmt)
return f.getvalue()
imagehandle = display.display(display.Image(data=showarray(env.render(mode='rgb_array')), width=450), display_id='gymscr')
action = 0
reward = 0
for _ in range(steps):
time.sleep(0.001)
action = agent.compute_action(observation, prev_reward=reward, prev_action=action)
observation, reward, done, info = env.step(action)
display.update_display(display.Image(data=showarray(env.render(mode='rgb_array')), width=450), display_id='gymscr')
if done: break
vdisplay.stop()
def __init__(self):
from xvfbwrapper import Xvfb
width = runtime.settings.SELENIUM_ARGS.get('WINDOW_SIZE_WIDTH')
height = runtime.settings.SELENIUM_ARGS.get('WINDOW_SIZE_HEIGHT')
self._vdisplay = Xvfb(width=width, height=height)
self._vdisplay.start()
super(Headless, self).__init__()
def xvfb():
with Xvfb():
display = os.environ["DISPLAY"]
if not can_connect_x11(display):
raise OSError("Xvfb did not come up")
yield
def __init__(self, log, config: ScraperConfig):
self.log = log
self.config = config
firefox_profile = webdriver.FirefoxProfile()
firefox_profile.set_preference('permissions.default.image',
self.config.img.get())
firefox_profile.set_preference(
'dom.ipc.plugins.enabled.libflashplayer.so', 'false')
if self.config.xvfb.get():
self.display = Xvfb()
self.display.start()
self._driver = webdriver.Firefox(firefox_profile)
self._driver.set_window_size(self.config.width.get(),
self.config.height.get())
self._driver.set_window_position(self.config.lat.get() or 0,
self.config.lon.get() or 0)
self.last_url = None
self.last_page = None
self._actions = {}
self._history = []
click = ClickAction(self, self.log)
filter_remove = FilterRemoveAction(self, self.log)
self.register('click', click)
self.register('remove', filter_remove)
def __init__(
self,
debugger: Debugger = Debugger.NONE,
use_profile: Optional[str] = None,
use_xvfb: bool = False,
working_path: Optional[str] = None,
):
# tokens used to notify log scanner to kill the browser process
self._abort_tokens: Set[Pattern[str]] = set()
self._bin_path: Optional[str] = None
self._checks: List[Union[CheckLogContents, CheckLogSize,
CheckMemoryUsage]] = list()
self._dbg = debugger
self._dbg_sanity_check(self._dbg)
self._launches = 0 # number of successful browser launches
self._logs = PuppetLogger(base_path=working_path)
self._proc: Optional["Popen[bytes]"] = None
self._profile_template = use_profile # profile that is used as a template
self._xvfb = None
self._working_path = working_path
self.profile: Optional[str] = None # path to profile
self.reason: Optional[Reason] = Reason.CLOSED
if use_xvfb:
if not system().startswith("Linux"):
self._logs.clean_up(ignore_errors=True)
raise OSError("Xvfb is only supported on Linux")
try:
self._xvfb = Xvfb(width=1280, height=1024)
except NameError:
self._logs.clean_up(ignore_errors=True)
raise OSError("Please install xvfbwrapper") from None
self._xvfb.start()
def get_apartment_data(url):
vdisplay = Xvfb()
vdisplay.start()
driver = webdriver.Chrome(options=options)
driver.set_window_size(1920, 1080)
driver.get(url)
city, district, street, block_number = get_address(driver)
sell_type, rent_info = get_selling_type(url)
if "продажа" in sell_type.lower():
rent_info = "Не аренда"
material, lift, year, rooms_number, floor, total_floors, total_area, kitchen_area, repair = get_apartment_params(driver)
block_type = "Вторичка"
living_area = "Не указано"
price = get_price(driver)
if "Аренда" in sell_type:
if "posutochno" in url:
price += "/день"
else:
price += "/мес."
#seller_type, seller_name = get_seller_info(driver)
images = get_photos(driver)
description = get_description(driver)
phone = get_seller_phone(driver)
selling_detail = "Не указано"
driver.quit()
vdisplay.stop()
return [city, district, street, block_number, sell_type, rent_info, price, block_type,
rooms_number, total_area, total_floors, material, selling_detail, images,
description, phone, kitchen_area, living_area, floor]
