def main():
parser = argparse.ArgumentParser()
parser.add_argument("--browser", default="firefox")
parser.add_argument("--debug", default=False, type=argparser_utils.boolean)
parser.add_argument("--sleep-multiplier",
default=True,
type=argparser_utils.boolean)
parser.add_argument("--push-to-server",
default=False,
type=argparser_utils.boolean)
parser.add_argument("--location",
type=argparser_utils.arg_location,
default="")
parser.add_argument("--virtual-display",
default=False,
type=argparser_utils.boolean)
args = parser.parse_args()
def start():
while True:
auto_swiper = AutoSwiper(email=Config.EMAIL,
password=Config.PASSWORD,
push_to_server=args.push_to_server,
sleep_multiplier=args.sleep_multiplier,
browser=args.browser,
debug=args.debug)
auto_swiper.start(args.location)
if args.virtual_display:
import pyvirtualdisplay
with pyvirtualdisplay.Display(visible=False, size=(2560, 1440)):
start()
else:
start()
def setUpClass(cls):
if HAS_VIRTUAL_DISPLAY:
cls.virtual_display = pyvirtualdisplay.Display(size=(1920, 1080))
cls.virtual_display.start()
super().setUpClass()
cls.selenium.set_window_size(1920, 1080)
def run_functional_tests(run_browsers_gui):
"""Run Functional Tests.
Args:
run_browsers_gui: Boolean to specify whether or not to run all functional tests in
the browser UI.
"""
os.environ['SERVER_SOFTWARE'] = 'Development'
os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
if run_browsers_gui:
argv = ['-W', 'tests/functional/']
nose.run(argv=argv)
return
try:
# start() assigns the DISPLAY environment variable to the virtual display.
virtual_display = pyvirtualdisplay.Display().start()
except easyprocess.EasyProcessCheckInstalledError:
virtual_display = None
if virtual_display:
argv = ['-W', 'tests/functional/']
nose.run(argv=argv)
virtual_display.stop()
else:
logging.warning(
'WARNING: You don\'t have Xvfb installed. This is required in order'
' to run tests in browsers headlessly')
logging.warning(
'You can either install xvfb ("sudo apt-get install xvfb" in Ubuntu)'
' or run tests with --browsers-gui switch')
argv = ['-W', 'tests/functional/']
nose.run(argv=argv)
def get_vuln_sites():
"""Wrapper method for get_dork_results.
Return:
- site_list: (type: VulnSite list) list of vulnerable site objects.
"""
site_list = []
display = pyvirtualdisplay.Display(visible=0, size=(1920, 1080))
display.start()
driver = start_webdriver()
for google_dork in SearchEntry.objects.filter(
category=SearchEntry.GoogleDork):
dork_results = get_dork_results(driver, google_dork.term)
if len(dork_results) > 0:
for dork_result in dork_results:
site_host = get_host_from_url(dork_result)
site_address = resolve_dns(site_host)
if site_address:
asn = resolve_asn(site_address)
site_object = VulnSite(site_host, site_address, asn,
dork_result, google_dork.reference,
'Google Dork')
site_list.append(site_object)
driver.quit()
display.stop()
return site_list
def launch():
id = flask.request.args.get('id')
if id == None:
return flask.redirect(flask.url_for('index'))
display = pyvirtualdisplay.Display(visible=0, size=(800, 600))
display.start()
browser = selenium.webdriver.Firefox()
browser.set_page_load_timeout(15)
browser.set_script_timeout(15)
browser.get('http://localhost:2000' + flask.url_for('go', id=id))
elema = browser.find_element_by_id('admin')
elemb = browser.find_element_by_id('bdmin')
elemc = browser.find_element_by_id('cdmin')
elemb.click()
elema.send_keys(key)
elemc.submit()
time.sleep(1)
browser.quit()
display.stop()
return "You're note is being serviced by an admin. Thanks for waiting!"
def setUpClass(cls):
cls.config = settings.FUNCT_TESTS_CONFIG if hasattr(
settings, 'FUNCT_TESTS_CONFIG') else None
super(FunctionalTestCase, cls).setUpClass()
if cls.config.get('VIRTUAL_DISPLAY'):
cls.virtual_display = pyvirtualdisplay.Display(
size=(cls.config.get('DISPLAY_WIDTH'),
cls.config.get('DISPLAY_HEIGHT')))
cls.virtual_display.start()
if cls.config.get('BROWSER') == 'FIREFOX':
from selenium.webdriver.firefox.webdriver import WebDriver
cls.selenium = WebDriver(
executable_path=cls.config.get('GECKO_DRIVER'))
cls.selenium.implicitly_wait(cls.config.get('DEFAULT_WAITING_TIME'))
cls.selenium.set_window_size(cls.config.get('DISPLAY_WIDTH'),
cls.config.get('DISPLAY_HEIGHT'))
if cls.config.get('HTML_REPORTS'):
cls.test_class_report = TestClassReport(name=cls.__name__)
cls.test_class_report.doc = cls.__doc__
cls.app_name = cls.__module__.split('.')[0].title()
if not StaticReportHAndler.current_app_report:
StaticReportHAndler.current_app_report = TestAppReport(
cls.app_name)
elif StaticReportHAndler.current_app_report.name != cls.app_name:
StaticReportHAndler.current_app_report.succeeded = all(
r.succeeded for r in
StaticReportHAndler.current_app_report.test_class_reports)
StaticReportHAndler.ran_tests.append(
StaticReportHAndler.current_app_report)
StaticReportHAndler.current_app_report = TestAppReport(
cls.app_name)
def test_login(self):
with pyvirtualdisplay.Display():
browser = Browser()
browser.visit("http://ui/accounts/login/")
browser.fill("login", "testuser")
browser.fill("password", "password")
browser.find_by_css(".btn-primary").click()
self.assertTrue(browser.find_by_text("Successfully signed in as testuser."))
def _initialize_display(self):
if not self.display:
self.display = pyvirtualdisplay.Display()
self.display.start()
if not self.browser:
self.browser = webdriver.Firefox()
self.browser.implicitly_wait(self.waiting_time_in_secs)
self.num_selenium_retrievers += 1
def run(self):
if 'pyvirtualdisplay' in globals():
display = pyvirtualdisplay.Display()
display.start()
self.run_firefox()
if 'pyvirtualdisplay' in globals():
display.stop()
def test_admin_login(self):
with pyvirtualdisplay.Display():
browser = Browser()
browser.visit("http://ui/admin/")
self.assertTrue(browser.find_by_text("Django administration"))
browser.fill("username", "sfmadmin")
browser.fill("password", "password")
browser.find_by_value("Log in").click()
self.assertTrue("Welcome" in browser.html)
def setUp(self):
if self.headless and self.browser != "PhantomJS":
disp_var = os.environ['DISPLAY']
import pyvirtualdisplay
if self.visible:
self.display = pyvirtualdisplay.Display(visible=1)
else:
self.display = pyvirtualdisplay.Display()
self.display.start()
if self.browser == "PhantomJS":
self.driver = webdriver.PhantomJS()
self.driver.set_window_size(1120, 550)
elif self.browser == "Chrome":
options = webdriver.ChromeOptions()
options.add_experimental_option(
"prefs", {"download.default_directory": os.getcwd()})
self.driver = webdriver.Chrome(chrome_options=options)
elif self.browser == "Firefox":
profile = webdriver.FirefoxProfile()
profile.set_preference("browser.download.folderList", 2)
profile.set_preference("browser.download.manager.showWhenStarting",
False)
profile.set_preference("browser.download.dir", os.getcwd())
profile.set_preference("browser.helperApps.neverAsk.saveToDisk",
'text/csv')
self.driver = webdriver.Firefox(firefox_profile=profile)
else:
self.driver = getattr(webdriver, self.browser)()
if self.headless and self.browser != "PhantomJS":
os.environ['DISPLAY'] = disp_var
self.driver.get(self.url)
self.actions = AC(self.driver)
self.sqlshare_login()
def start(self):
self._virtual_display = pyvirtualdisplay.Display(
backend='xvfb',
size=(self.width, self.height),
color_depth=self.colordepth,
use_xauth=self.xauth,
extra_args=self.args)
self._virtual_display.start()
self.display = self._virtual_display.display
assert self._virtual_display.is_alive()
def _run(display_size, queue, stop_event):
import pyvirtualdisplay
import os
os.setpgrp()
pydisp = pyvirtualdisplay.Display(visible=0, size=display_size)
display = pydisp.start().display
queue.put(display)
stop_event.wait()
pydisp.stop()
queue.put(None)
def init_driver():
if 'microsoft' in platform.platform().lower():
# You're running in Windows Subsystem for Linux,
# so use xvfb and pyvirtualdisplay to simulate headless mode
import pyvirtualdisplay
display = pyvirtualdisplay.Display(visible=0, size=(1920, 1080))
display.start()
else:
# Otherwise just run headless mode
os.environ['MOZ_HEADLESS'] = '1'
return webdriver.Firefox()
def __init__(self, configmananger):
display = pyvirtualdisplay.Display(visible=0, size=(
1280,
1024,
))
display.start()
self.driver = webdriver.Firefox(self.get_profile())
self.config = configmananger.get_config('natwest')
self.main()
time.sleep(2)
self.driver.quit()
display.stop()
def start(self):
self._virtual_display = pyvirtualdisplay.Display(
backend='xvfb',
size=(self.width, self.height),
color_depth=self.colordepth,
use_xauth=self.xauth)
self._virtual_display.cmd.extend(self.args)
self._virtual_display.start()
self.display = self._virtual_display.display
if not self._virtual_display.is_alive():
ret = self._virtual_display.return_code
raise XvfbExitedError("Xvfb exited with exit code {0}".format(ret))
def run_js_tests(run_browsers_gui):
"""Run JS tests suite.
If run_browsers_gui is True, then run all JS tests in all browsers found in
the system, showing their UI during the tests.
If run_browsers_gui is False, then:
a) if Xvfb is present in the system, JS tests for all browsers in the system
are run in the virtual buffer, so no browser GUI is shown.
b) If Xvfb is not available, then JS tests are run headlessly only in
PhantomJS.
Args:
run_browsers_gui: Boolean to specify whether or not to run all JS tests in
all browsers available in the system showing their UI.
"""
# TODO(mario): this is necessary since testem doesn't have any facility to run
# its server from a random port for CI. If multiple users are running tests in
# the same machine at the same time it crashes.
# This can be removed once https://github.com/airportyh/testem/issues/283 is
# fixed.
port = get_random_available_port()
if run_browsers_gui:
subprocess.call('node ./node_modules/testem/testem.js -p %s ci' % port,
env=get_js_tests_environment(),
shell=True)
return
try:
# start() assigns the DISPLAY environment variable to the virtual display.
virtual_display = pyvirtualdisplay.Display().start()
except easyprocess.EasyProcessCheckInstalledError:
virtual_display = None
if virtual_display:
subprocess.call('node ./node_modules/testem/testem.js -p %s ci' % port,
env=get_js_tests_environment(),
shell=True)
virtual_display.stop()
else:
print(
'WARNING: You don\'t have Xvfb installed. This is required in order'
' to run tests in browsers headlessly')
print(
'You can either install xvfb ("sudo apt-get install xvfb" in Ubuntu)'
' or run tests with --browsers-gui switch')
subprocess.call(
'node ./node_modules/testem/testem.js -l phantomjs -p %s ci' %
port,
env=get_js_tests_environment(),
shell=True)
def start(self):
self._virtual_display = pyvirtualdisplay.Display(
backend='xvfb', size=(self.width, self.height),
color_depth=self.colordepth, use_xauth=self.xauth)
self._virtual_display.cmd.extend(self.args)
self._virtual_display.start()
self.display = self._virtual_display.display
if not self._virtual_display.is_alive():
ret = self._virtual_display.return_code
self._virtual_display.wait() # collect outputs
stdout = self._virtual_display.stdout
stderr = self._virtual_display.stderr
raise XvfbExitedError("Xvfb exited with exit code {0}\nXvfb stdout:\n {1}\nXvfb stderr:\n {2}".format(
ret,
'\n '.join(stdout.splitlines()),
'\n '.join(stderr.splitlines())))
def __init__(self,
url=None,
driver=None,
headless_display=False,
extra_js_files=[],
disable_cleanup=False,
width=1366,
height=768,
auto_refresh=False,
load_no_images=False,
custom_proxy=None,
timeout=None):
xcontext.current_xbrowser = self
self._js_filenames = JS_FILENAMES + extra_js_files
if not settings.DEBUG:
self.disable_cleanup = False
else:
self.disable_cleanup = disable_cleanup
self.auto_refresh = auto_refresh
self.auto_visited_domains = set()
self.display = None
if headless_display:
self.display = pyvirtualdisplay.Display(visible=0,
size=(width, height))
self.display.start()
self.driver = None
self.driver_owned = False
if driver:
self.driver = driver
self.driver_owned = False
else:
self.driver = seleniumtools.create_default_driver(
load_no_images=load_no_images, custom_proxy=custom_proxy)
self.driver_owned = True
if isinstance(timeout, int):
self.driver.set_page_load_timeout(timeout)
self.url = None
if url:
self.load_url(url)
def __init__(self, socket_timeout=SOCKET_TIMEOUT, *args, **kwargs):
super(SeleniumWorker, self).__init__(*args, **kwargs)
# set up the xvfb display
self.display = pyvirtualdisplay.Display(visible=0, size=(800, 800))
self.display.start()
# NOTE: This DOES change the socket timeout globally. There's
# not much we can do about that though, since we don't have
# access to the offending socket buried deeply in urllib2.
# set socket timeout to kill hung chromedriver connections
self.original_socket_timeout = socket.getdefaulttimeout()
socket.setdefaulttimeout(socket_timeout)
# Set up the webdriver options
self.options = webdriver.ChromeOptions()
self.options.add_argument('--start-maximized')
self.options.add_argument('--disable-java')
self.options.add_argument('--incognito')
self.options.add_argument('--use-mock-keychain')
def _initialize_driver(self):
self.virt_display = pyvd.Display(visible=0, size=(1600, 900))
self.virt_display.start()
browser_data_dir = os.path.join(os.getcwd(), 'browser_data')
if os.path.exists(browser_data_dir):
shutil.rmtree(browser_data_dir, ignore_errors=True)
self.download_dir = os.path.abspath(
os.path.join(browser_data_dir, 'driver_downloads'))
sys.stdout.write('Starting a ChromeDriver ')
sys.stdout.write('with the default download directory:\n')
sys.stdout.write(' "{}"'.format(self.download_dir))
sys.stdout.write('...\n')
_options = webdriver.ChromeOptions()
# using to --no-startup-window to run Chrome in the background throws a
# WebDriver.Exception with "Message: unknown error: Chrome failed to
# start: exited normally"
##_options.add_argument('--no-startup-window ')
_options.add_argument('user-data-dir={}'.format(browser_data_dir))
prefs = {'download.default_directory': self.download_dir}
_options.add_experimental_option("prefs", prefs)
return webdriver.Chrome(chrome_options=_options)
def __init__(self, name=None, width=1024, height=768):
if not len(displays):
display = pyvirtualdisplay.Display(visible=False, size=(2048, 768))
display.start()
displays.append(display)
profile = webdriver.FirefoxProfile()
# profile.add_extension(
# extension="/home/gaea/canopy/var/identities/"
# "6c189616-4fe1-4f3f-84dc-c4a13ee9b155/"
# "asteria/asteria-dev.xpi"
# )
binary = "/home/gaea/firefox/firefox-bin"
self.browser = webdriver.Firefox(firefox_profile=profile,
firefox_binary=binary)
count = len(browsers)
browsers.append(self)
self._top = 0
self._left = count * 1024
self._width = width
self._height = height
self._update_window()
self.name = name
self.shot_id = 0
for line in process.stdout:
print(line.decode().strip())
for i in pip_requirements:
run_subprocess_command("pip install " + i)
for i in debian_requirements:
run_subprocess_command("apt install " + i)
# Setup virtual display for Google colab
if "google.colab" in sys.modules:
import pyvirtualdisplay
_display = pyvirtualdisplay.Display(
visible=False, # use False with Xvfb
size=(1400, 900))
_ = _display.start()
import numpy as np
import time
import imageio # To render episodes in GIF images (otherwise there would be no render on Google Colab)
# C.f. https://stable-baselines.readthedocs.io/en/master/guide/examples.html#bonus-make-a-gif-of-a-trained-agent
# To display GIF images in the notebook
import IPython
from IPython.display import Image
import tensorflow as tf
from tf_agents.agents.reinforce import reinforce_agent
from tf_agents.drivers import py_driver
from tf_agents.environments import suite_gym
from tf_agents.environments import tf_py_environment
from tf_agents.networks import actor_distribution_network
from tf_agents.policies import py_tf_eager_policy
from tf_agents.replay_buffers import reverb_replay_buffer
from tf_agents.replay_buffers import reverb_utils
from tf_agents.specs import tensor_spec
# from tf_agents.trajectories import trajectory
from tf_agents.utils import common
# Set up a virtual display for rendering OpenAI gym environments.
display = pyvirtualdisplay.Display(visible=0, size=(1400, 900)).start()
env_name = "CartPole-v0" # @param {type:"string"}
num_iterations = 250 # @param {type:"integer"}
collect_episodes_per_iteration = 2 # @param {type:"integer"}
replay_buffer_capacity = 2000 # @param {type:"integer"}
fc_layer_params = (100, )
learning_rate = 1e-3 # @param {type:"number"}
log_interval = 25 # @param {type:"integer"}
num_eval_episodes = 10 # @param {type:"integer"}
eval_interval = 50 # @param {type:"integer"}
env = suite_gym.load(env_name)
def setUpClass(cls): cls.display = pyvirtualdisplay.Display(visible=False, size=(1400, 900)) cls.display.start()
if not os.path.exists(screenshots_path):
os.makedirs(screenshots_path)
visible = '-v' in sys.argv
def report_error():
exc = traceback.format_exc()
syslog.syslog(syslog.LOG_ERR, "DynDNS bot stopped working!")
for line in exc.split('\n'):
syslog.syslog(syslog.LOG_ERR, line)
sys.stderr.write(exc)
try:
display = pyvirtualdisplay.Display(visible=visible, size=(1024, 768))
display.start()
except:
report_error()
sys.exit(1)
try:
# Create a new instance of the Firefox driver
driver = webdriver.Firefox()
except:
report_error()
sys.exit(1)
try:
# Log in DynDNS
driver.get("https://account.dyn.com/entrance/")
def google(export_bool=False, output_bool=False):
dorks = {
"Pony": [
"intitle:Authorization inurl:panel*/admin.php intext:Authorization. Sign in.",
"intitle:Authorization inurl:panel*/*admin.php",
"intitle:Authorization inurl:*admin.php Authorization. User Password Save password. Login. TF."
],
"WannaCry":
"intitle:\"index of\" \"@[email protected]",
"Stealer":
"intitle:\"(c) Bilal Ghouri\"",
"LokiBot":
"inurl:PvqDq929BSx_A_D_M1n_a.php intitle:Auth",
"1ms0rry":
"intitle:1ms0rry MINERPANEL",
"SpyEye":
"intitle:FRMCP intext:Please, enter password"
}
print(bcolors.OKBLUE + "++++++++++++++++++++++++++++++++++")
print("Google dorks")
print("++++++++++++++++++++++++++++++++++" + bcolors.ENDC)
info = {}
links_list = []
display = pyvirtualdisplay.Display(visible=0, size=(800, 600))
display.start()
directory = "google" + timestamp_now
# now Firefox will run in a virtual display.
# you will not see the browser.
browser = webdriver.Firefox()
for i in dorks.keys(): # for every dork in dictionary
if i == "Pony": # for Pony is more than one dork
for j in dorks[i]:
browser.get('http://www.google.com/search?q=' + j +
"&t=h_&ia=web")
links = browser.find_elements_by_xpath("//h3//a[@href]")
for elem in links:
link = elem.get_attribute("href")
links_list.append(link)
info.update({i: links_list})
links_list = []
else:
browser.get('http://www.google.com/search?q=' + dorks[i] +
"&t=h_&ia=web")
links = browser.find_elements_by_xpath("//h3//a[@href]")
for elem in links:
link = elem.get_attribute("href")
links_list.append(link)
info.update({i: links_list})
links_list = []
if export_bool:
try:
export(directory, info)
print("Exported to " + directory + '.txt')
except OSError as e:
print("Error " + e.strerror)
elif output_bool:
for i in info:
print("-----------------------------")
print(i)
for j in info[i]:
print(j)
print("-----------------------------")
else:
print("Please add --output or --export")
browser.quit()
display.stop()
from model import Combiner_network, GVF_network, soft_update
from utils import choose_action, concat_state_action, exp_sample, get_feature
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
env = gym.make("CartPole-v1")
seed = 0
np.random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed(seed)
env.seed(seed)
state = env.reset()
done = False
_display = pyvirtualdisplay.Display(visible=False, size=(1400, 900))
_ = _display.start()
n_action = env.action_space.n
n_state = len(env.reset())
n_input = n_state + n_action
n_feature = 8
n_output = 1
memory = deque(maxlen=200000)
lr_f = 1e-5
lr_c = 1e-5
reward_discount_factor = 0.99
GVF_discount_factor = 0.99
def test_home(self):
with pyvirtualdisplay.Display():
browser = Browser()
browser.visit("http://ui")
def main():
if args.use_virtual_display:
import pyvirtualdisplay
display = pyvirtualdisplay.Display(visible=0, size=(800, 600))
display.start()
scraper = ArtistWorkScraper(fetch_extras=args.fetch_extras,
use_firefox=args.use_firefox)
scraper.login_to_artistworks(username=args.username,
password=args.password)
if args.only_lessons:
lesson_ids = args.only_lessons
department_name = 'Misc lessons'
else:
department_name = scraper.get_department_name(args.department)
lessons = scraper.get_all_lesson_ids_for_department(args.department)
lesson_ids = lessons.keys()
# Create nice txt file with lessons info
output_path = Path(args.output_dir).joinpath(
args.root_folder).joinpath(department_name)
os.makedirs(str(output_path), exist_ok=True)
f = Path(args.output_dir).joinpath(args.root_folder).joinpath(
department_name).joinpath('Lessons.txt').open('w')
f.write('Lessons for {}'.format(department_name))
f.write('\r\n')
for i, lesson in enumerate(lessons):
f.write('{id}. {name}'.format(id=i, name=lessons[lesson]))
f.write('\r\n')
f.close()
lessons_db = shelve.open(
os.path.join(args.output_dir, department_name + '_lessons.db'))
masterclasses_db = shelve.open(
os.path.join(args.output_dir, department_name + '_masterclasses.db'))
for lesson_id in lesson_ids:
if lesson_id not in lessons_db:
lessons_db[lesson_id] = scraper.get_lesson_by_id(lesson_id)
if args.fetch_masterclasses:
for masterclass_id in lessons_db[lesson_id].masterclass_ids:
if masterclass_id not in masterclasses_db:
masterclasses_db[
masterclass_id] = scraper.get_masterclass_by_id(
masterclass_id)
# start downlaoding
downloader = AsyncDownloader()
for lesson in lessons_db.values():
lesson_output_folder_path = Path(args.output_dir).joinpath(
args.root_folder).joinpath(department_name).joinpath(
get_valid_filename(lesson.name))
for lesson_link in lesson.links:
downloader.download_link(lesson_link, lesson_output_folder_path)
if args.fetch_masterclasses:
for masterclass_id in lesson.masterclass_ids:
masterclass = masterclasses_db[masterclass_id]
masterclass_output_folder_path = lesson_output_folder_path.joinpath(
get_valid_filename(masterclass.name))
if masterclass_output_folder_path.exists():
masterclass_output_folder_path.with_name(
masterclass_output_folder_path.name +
'masterclass_{}'.format(masterclass_id))
for masterclass_link in masterclass.links:
downloader.download_link(masterclass_link,
masterclass_output_folder_path)
downloader.run()
unite_ts_videos(os.path.join(args.output_dir, args.root_folder))
scraper.exit()
