def setUpClass(cls):
skipIf(config.get("no_selenium"), "Told not to run selenium tests")
ffp = FirefoxProfile()
ffp.native_events_enabled = True
cls.selenium = WebDriver(ffp)
super(JavascriptTests, cls).setUpClass()
def __init__(self):
self.start_urls = ['http://www.amazon.com/review/top-reviewers?page=' + str(self.start)]
self.allowed_domains = ['www.amazon.com']
# self.profile = FirefoxProfile('/home/romitas/.mozilla/firefox/68h3udd9.AmazonScraper')
profile = FirefoxProfile()
profile.set_preference('network.protocol-handler.external.mailto', False)
self.driver = webdriver.Firefox(self.profile)
self.driver_login()
def setUpClass(cls):
if config.get("no_selenium"):
raise SkipTest("Told not to run selenium tests")
ffp = FirefoxProfile()
ffp.native_events_enabled = True
cls.selenium = WebDriver(ffp)
super(JavascriptTests, cls).setUpClass()
def __init__(self, category=None, *args, **kwargs):
super(DmozSpider, self).__init__(*args, **kwargs)
self.selection = [kwargs.get('selection')][0]
self.logger.info("Selection is: %s", str(self.selection))
if self.selection != 'active' and self.selection != 'retired':
sys.exit("Invalid selection")
elif self.selection == None:
sys.exit("Selection not set")
self.output_dir = [kwargs.get('output_dir')][0]
self.logger.info("Output directory is: %s", str(self.output_dir))
if self.output_dir == None:
sys.exit("No output directory specified")
self.position_type = [kwargs.get('position_type')][0]
self.logger.info("Position type is: %s", str(self.position_type))
if self.position_type != 'qb' and self.position_type != 'wr' and self.position_type != 'rb' and self.position_type != 'te':
sys.exit("Invalid position type")
collect_csvs = [kwargs.get('collect_csvs')][0]
self.logger.info("Collect csvs is set to: %s", str(collect_csvs))
if collect_csvs == None:
sys.exit("Collect csvs is not set!")
elif collect_csvs == "True":
self.collect_csvs = True
elif collect_csvs == "False":
self.collect_csvs = False
else:
sys.exit("Collect csvs set to invalid value")
if self.selection == 'retired':
self.old_guys = [kwargs.get('old_guys')][0]
self.logger.info("Old Guys is set to %s", str(self.old_guys))
if self.old_guys == None:
sys.exit("Old guys not set!")
elif self.old_guys != "True" and self.old_guys != "False":
sys.exit("Old guys set to invalid value!")
# If they had college stats
self.gameLogsXPathCollege = "//*/*[1]/*/*[3]/a[contains(.,'Gamelogs')]"
# No college stats
self.gameLogsXPathNoCollege = "//*/*[1]/*/*[2]/a[contains(.,'Gamelogs')]"
# Initialize a new Firefox webdriver
profile = FirefoxProfile("/Users/andrasta/Desktop/InfoVis/project/FirefoxProfileUpdated")
profile.set_preference('browser.download.folderList', 2)
profile.set_preference('browser.download.dir', os.getcwd()+"/"+self.output_dir) #os.getcwd()+"/QB")
profile.set_preference('browser.helperApps.neverAsk.saveToDisk', 'text/csv')
profile.set_preference('browser.helperApps.neverAsk.openFile', 'text/csv')
self.driver = webdriver.Firefox(firefox_profile=profile)
# Doesn't matter
# self.driver.maximize_window()
self.wait = WebDriverWait(self.driver, 5)
self.no_stat_players = []
def __init__(self, profile, userAgent=''):
self.profile = FirefoxProfile(profile)
if (userAgent != ''):
self.profile.set_preference("general.useragent.override",
userAgent)
self.driver = webdriver.Firefox(self.profile)
self.driver.execute_script(
"Object.defineProperty(navigator, 'webdriver', {get: () => undefined})"
)
self.driver.maximize_window()
wait = WebDriverWait(self.driver, 10)
def login_direct(
self,
url="http://ds01za003:7813/lscapps/eric-emi-murali/AutoLoginServlet",
profile_path=""):
"""Alternative login that uses firefox profile with Modify Headers
to access Eric without using the portal (doesn't work in every
Eric instance but does with DEV10).
Args:
url - "auto login" url
profile_path - path to Firefox profile with apporpriate Modify
Headers details.
"""
ffp_object = FirefoxProfile(profile_path)
# Use Webdriver to open Firefox using chosen profile
self.driver = webdriver.Firefox(ffp_object)
# Open the URL
self.driver.get(url)
# Check expected page is present
WebDriverWait(
self.driver,
20).until(lambda driver: "reports found for" in driver.page_source
or "0 report(s) found for user" in driver.page_source)
# Wait for "please wait" message to go
WebDriverWait(self.driver,
20).until(lambda driver: not self.check_page_blocked())
def __init__(self,userName,profilePath):
profile = FirefoxProfile(profilePath)
driver = webdriver.Firefox(profile)
driver.get("https://web.whatsapp.com/")
driver.implicitly_wait(15)
self.driver = driver
self.userName = userName
def start_driver(self, in_background=False, implicit_wait_time=4):
"""
Start the selenium web-driver.
:return: the selenium web-driver object, from which you will find elements, click things, etc.
"""
# Suppress actual physical browser opening, put in background.
options = Options()
if in_background:
options.add_argument('--headless')
if self.browser == 'firefox':
profile = FirefoxProfile(self.browser_profile_path)
driver = webdriver.Firefox(
profile,
options=options,
executable_path=self.browser_driver_path)
elif self.browser == 'chrome':
print("Haven't implemented Chrome profiles yet. TODO.")
#profile = ...
#driver = webdriver.Chrome()
else:
print(
"Browser argument not recognized. Must be one of: ['firefox', 'chrome']."
)
return None
driver.implicitly_wait(implicit_wait_time)
return driver
def __enter__(self):
options = Options()
options.headless = self.headless
#options.log.level = "debug"
driver_path = Path(f"{os.path.dirname(__file__)}/../../geckodriver")
if not os.path.exists("/tmp/flappy_ai_profile"):
print(
f"Please make a profile as seen https://stackoverflow.com/questions/22685755/retaining-cache-in-firefox-and-chrome-browser-selenium-webdriver"
)
print(
"Firefox will not cache DNS and will cause very long load times unless you do this."
)
print("Save the profile under /tmp/flappy_ai_profile")
raise Exception
profile = FirefoxProfile("/tmp/flappy_ai_profile")
self._browser = webdriver.Firefox(options=options,
executable_path=driver_path,
firefox_profile=profile)
#driver_path = Path(f"{os.path.dirname(__file__)}/../../chromedriver")
#self._browser = webdriver.Chrome(options=options, executable_path=driver_path)
self._browser.set_window_size(self._browser_width,
self._browser_height)
self._browser.get("https://flappybird.io/")
self._game_element = self._browser.find_element_by_id(id_="testCanvas")
pos = self._browser.get_window_position()
self._pos_x = pos["x"]
self._pos_y = pos["y"]
return self
def __init__(self):
self.articles = []
self.query = None
try:
profile = FirefoxProfile(FF_PROFILE_PATH)
self.firefox = webdriver.Firefox(profile)
except Exception, e:
self.firefox = webdriver.Firefox()
def session():
options = webdriver.FirefoxOptions()
profile = FirefoxProfile(
r'/Users/Luca/Library/Application Support/Firefox/Profiles/s8i83kl9.default'
)
driver = webdriver.Firefox(
firefox_profile=profile,
options=options,
executable_path="/Users/luca/Desktop/zoe-ocado/geckodriver")
driver.get("https://accounts.ocado.com/auth-service/sso/login")
return driver
def scrap_each_query(keyword, date, start, end, page, out_file):
real_keyword = keyword
keyword = urllib.parse.quote(urllib.parse.quote(keyword))
profile = FirefoxProfile(
"/Users/Rock_Wang/Library/Application Support/Firefox/Profiles/038xacgn.default-release"
)
driver = webdriver.Firefox(profile)
url = base_url + keyword + "&typeall=1&suball=1×cope=custom:" + start + ":" + end + "&page=" + "1"
driver.get(url)
systime.sleep(5)
count = 0
print("crawling data on {} and using keyword {}".format(date, keyword))
for i in range(int(page)):
url = base_url + keyword + "&typeall=1&suball=1×cope=custom:" + start + ":" + end + "&page=" + str(
i + 1)
driver.get(url)
page_source = driver.page_source
soup = BeautifulSoup(page_source, "html.parser")
for card in soup.find_all('div', class_='card'):
content = card.find_all('div', class_='content')
card_act = card.find_all('div', class_='card-act')
# check if forward
if check_forward(content):
continue
for c in content:
p_txt = c.find_all('p', class_='txt')[0].text
txt = p_txt.replace('\n', '').strip()
# print(txt)
p_time = c.find_all('p', class_='from')[0]
time = p_time.find('a').contents[0]
time = time.replace('\n', '').strip()
# print(time)
for act in card_act:
keep, forward, comment, like = get_four_stats(act)
# print("{} {} {}".format(forward, comment, like))
# print(count)
count = count + 1
with open(out_file, 'ab+') as write_obj:
# Create a writer object from csv module
csv_writer = csv.writer(write_obj)
# Add contents of list as last row in the csv file
csv_writer.writerow([
count, real_keyword, date, txt, time, forward, comment,
like
])
print("Finished writing page {}".format(i))
driver.close()
write_obj.close()
def run_test():
profile = FirefoxProfile("/Users/andrasta/Desktop/InfoVis/project/FirefoxProfile")
profile.set_preference("browser.download.folderList", 2)
profile.set_preference("browser.download.dir", os.getcwd() + "/QB")
profile.set_preference("browser.helperApps.neverAsk.saveToDisk", "text/csv")
profile.set_preference("browser.helperApps.neverAsk.openFile", "text/csv")
driver = webdriver.Firefox(firefox_profile=profile)
# Doesn't matter
# self.driver.maximize_window()
wait = WebDriverWait(driver, 5)
driver.maximize_window()
driver.get("http://www.pro-football-reference.com/players/B/BreeDr00/gamelog//")
csvXPath = "(//span[contains(.,'Export')])[1]"
csvLink = wait.until(EC.element_to_be_clickable((By.XPATH, csvXPath)))
print "XPATH found link text is: ", csvLink.text
if csvLink != None and csvLink.text == "Export":
# ActionChains(driver).move_to_element(csvLink).perform()
ActionChains(driver).click(csvLink).perform()
def setUp(self):
from selenium.webdriver.firefox.webdriver import FirefoxProfile
fp = FirefoxProfile()
fp.set_preference("browser.download.folderList", 2)
fp.set_preference("browser.download.dir", self.DOWNLOAD_DIRECTORY)
fp.set_preference("browser.helperApps.neverAsk.saveToDisk", "text/html, application/zip, text/plain, application/force-download, application/x-tar")
self.selenium = WebDriver(firefox_profile=fp)
self.selenium.implicitly_wait(1) # wait one second before failing to find
# create the download dir
if not os.path.exists(self.DOWNLOAD_DIRECTORY):
os.makedirs(self.DOWNLOAD_DIRECTORY)
def make_driver():
driver = None
if config.use_proxy and config.browser != 'chrome':
raise Exception(f'proxy only supported in chrome')
if config.browser == 'chrome':
_chrome_options = chrome_options()
_chrome_options.add_argument("--disable-extensions")
_chrome_options.add_argument("--disable-gpu")
_chrome_options.add_argument("--disable-notifications")
_chrome_options.add_argument("--disable-logging")
_chrome_options.add_experimental_option('excludeSwitches', ['enable-logging'])
_chrome_options.add_experimental_option('prefs', {
'prefs': {'intl.accept_languages': 'en,en-US'},
'profile.default_content_setting_values.notifications' : 2,
'profile.managed_default_content_settings.images': 2})
#_chrome_options.add_argument(f"user-data-dir={config.chrome_profile_path}")
if config.use_proxy:
host, port, protocol = config.proxy_list[0]
_chrome_options.add_argument(f"--proxy-server={protocol}://{host}:{port}")
helper.thread_safe_print(f'using proxy: {protocol}://{host}:{port}')
if config.headless:
_chrome_options.add_argument("--start-maximized")
_chrome_options.add_argument("--headless")
driver = webdriver.Chrome(executable_path=config.chrome_driver, options=_chrome_options)
elif config.browser == 'firefox':
_firefox_profile = FirefoxProfile(config.firefox_profile_path)
_firefox_options = firefox_options()
_firefox_options.headless = config.headless
driver = webdriver.Firefox(_firefox_profile, options=_firefox_options)
else:
raise Exception(f'unknown brower "{config.browser}"')
if not config.skip_login:
login(driver)
return driver
def openSite(filename):
profile = FirefoxProfile("/home/squid/proxyprofile/")
driver = webdriver.Firefox(firefox_profile=profile)
associated = []
with open(filename) as f:
address = f.readline()
while address:
driver.get(address)
for a in driver.find_elements_by_xpath("//a[@href]"):
associated.append(a.get_attribute("href"))
if len(associated) == 3:
break
for ap in associated:
driver.get(ap)
associated = []
address = f.readline()
driver.close()
def __init__(self, headless=False):
self.profile = FirefoxProfile(self.firefox_profile_dir)
self.options = Options()
self.options.add_argument('-headless')
if headless:
self.driver = Firefox(firefox_profile=self.profile, executable_path='geckodriver', options=self.options)
else:
self.driver = Firefox(firefox_profile=self.profile, executable_path='geckodriver')
self.driver.get('https://web.whatsapp.com')
page_not_loaded = True
while page_not_loaded:
try:
self.searchbox = self.driver.find_element_by_class_name(self.whatsapp_searchbox_class)
page_not_loaded = False
except NoSuchElementException:
time.sleep(0.5)
def sign_into_venmo(headless):
"""Return web driver signed into Venmo."""
# Create browser driver. Note, needs to be headless to run via CLI.
# The profile is needed because Venmo will need to recognize your device to
# avoid two-factor authentication.
profile = FirefoxProfile(PROFILE_PATH)
options = Options()
options.headless = headless # needed to run via CLI
driver = webdriver.Firefox(profile, options=options)
# Fill in login credentials.
driver.get('https://venmo.com/account/sign-in')
inputs = driver.find_elements_by_css_selector('input.auth-form-input')
button = driver.find_element_by_css_selector('button.ladda-button')
inputs[0].send_keys(LOGIN)
inputs[1].send_keys(PASSWORD)
button.click()
return driver
def setUp(self):
StaticLiveServerTestCase.setUp(self)
profile = FirefoxProfile()
# Browser itself attempts to validate form fields before they are sent to django.
# Fields where input type="Number" accept "100.0" when locale is "en" and "100,0" when locale is "fi", and when
# they reject the value, django sees an empty value instead.
# To prevent this causing more problems, force browser used by tests to use same locale as django, typically
# "en".
# We may want to occassionally test with other locales, so localize_input, number_format etc. when entering
# and reading decimals/floats.
profile.set_preference("intl.accept_languages", get_language())
profile.set_preference("general.useragent.locale", get_language())
self.selenium = Firefox(
firefox_profile=profile,
executable_path='node_modules/geckodriver/geckodriver')
self.selenium.maximize_window()
def init_doge():
try:
info_wb = openpyxl.load_workbook(filename=r'data\crane_stats.xlsx')
info_sheet = info_wb['INFO']
except Exception as err:
error_hook.fatal_err(
"Не могу получить доступ к crane_stats.xlsx/INFO. "
"Убедитесь в наличии файла 'crane_stats.xlsx' в папке 'data' и повторите попытку."
"\n Файл должен быть закрыт!", err)
return None
dogeco_cap_img = 'dgccap.png'
if not info_sheet.cell(row=10, column=2).value:
error_hook.warn_err(
"В ячейке 'B10' не указан пароль. Работа с сайтом fredoge.co.in невозможна"
)
return None
else:
dogeco_pass = str(info_sheet.cell(row=10, column=2).value)
if not info_sheet.cell(row=19, column=9).value:
error_hook.warn_err(
"В ячейке 'i19' не указан ключ каптчи. Работа с сайтом fredoge.co.in невозможна"
)
return None
else:
dogeco_cap_api = str(info_sheet.cell(row=19, column=9).value)
if not info_sheet.cell(row=20, column=9).value:
error_hook.warn_err(
"В ячейке 'i20' не указан профиль. Работа с сайтом fredoge.co.in невозможна"
)
return None
else:
profile_dgc = FirefoxProfile(
str(info_sheet.cell(row=20, column=9).value))
dogeco_user = FreeDOGECO(dogeco_mail, dogeco_pass, profile_dgc,
dogeco_cap_img, dogeco_cap_api)
return dogeco_user
def setUp(self):
self.output_formats = GlobalParameterFactory.create(parameter_name='output_formats', parameter_value=LiveServerTest.OUTPUT_FORMATS)
from selenium.webdriver.firefox.webdriver import FirefoxProfile
fp = FirefoxProfile()
fp.set_preference("browser.download.folderList", 2)
fp.set_preference("browser.download.dir", self.DOWNLOAD_DIRECTORY)
fp.set_preference("browser.helperApps.neverAsk.saveToDisk", "text/html, application/zip, text/plain, application/force-download, application/x-tar")
self.selenium = WebDriver(firefox_profile=fp)
self.selenium.implicitly_wait(1) # wait one second before failing to find
# create the download dir
if not os.path.exists(self.DOWNLOAD_DIRECTORY):
os.makedirs(self.DOWNLOAD_DIRECTORY)
# ensure that it is empty
for root, dirs, files in os.walk(self.DOWNLOAD_DIRECTORY):
for file in files:
os.remove(os.path.join(root, file))
self.halt_on_exception = os.environ.get('TAO_HALT_ON_EXCEPTION', False) == 'True'
return
def scrap_each_query(keyword, date, start, end, page, query):
real_keyword = keyword
keyword = urllib.quote(urllib.quote(keyword))
# login_url = 'http://m.weibo.com/'
# driver = webdriver.Chrome()
# driver.get(login_url)
# driver.implicitly_wait(2)
# string = '登录'
# driver.find_element_by_link_text ( string.decode('utf-8') ).click()
# driver.implicitly_wait(2)
# driver.find_element_by_link_text(string.decode('utf-8') ).click()
# savedCookies = driver.get_cookies()
# # login code
# pickle.dump(driver.get_cookies() , open("chrome.pkl","wb"))
# driver.close()
all_content = []
all_time = []
profile = FirefoxProfile(
"/Users/xuzhouyin/Library/Application Support/Firefox/Profiles/ky3euswa.default-1457644506160"
)
driver = webdriver.Firefox(profile)
# co = webdriver.ChromeOptions()
# co.add_argument('user-data-dir=/Users/xuzhouyin/Library/Application Support/Google/Chrome/')
# driver = webdriver.Chrome(chrome_options = co)
url = base_url + keyword + "&typeall=1&suball=1×cope=custom:" + start + ":" + end + "&page=" + "1"
driver.get(url)
systime.sleep(5)
for i in range(int(page)):
url = base_url + keyword + "&typeall=1&suball=1×cope=custom:" + start + ":" + end + "&page=" + str(
i + 1)
# url = "http://s.weibo.com/weibo/%25E5%2585%2583%25E6%2597%25A6&typeall=1&suball=1×cope=custom:2016-12-31:2016-12-31&Refer=g"
# chrome_options = Options()
# chrome_options.add_argument("~/Library/Application Support/Google/Chrome/Default");
# co = webdriver.ChromeOptions()
# co.add_argument('/Users/xuzhouyin/Library/Application\ Support/Google/Chrome/Default')
# for cookie in pickle.load(open("chrome.pkl", "rb")):
# driver.add_cookie(cookie)
driver.get(url)
# driver.magage().add_cookie(savedCookies)
page_source = driver.page_source
soup = BeautifulSoup(page_source, "html.parser")
content = soup.findAll("p", {"class": "comment_txt"})
time = soup.findAll("a", {"class": "W_textb"})
for each in content:
all_content.append(each.get_text().encode('utf-8'))
for each in time:
each = each.get_text()
each = each.encode('utf-8')
time = ""
if "月" in each:
time = str(datetime.datetime.now().year
) + "-" + each[0:each.index("月")] + "-" + each[
(each.index("月") + 3):each.index("日")]
else:
time = each[0:each.index(" ")]
all_time.append(time)
driver.close()
save_to_csv(file + str(file_index), real_keyword, date, all_content,
all_time, query)
def __init__(self, input_file):
# print "input file: " + input_file
self.stage = 0 # use stage to see where in the process you are so you know where to come back to.
self.whoops = (
0
) # number of mistakes on this particular's entry's upload. you can change number of mistakes allowed in webFailure function.
self.volume = Volume()
self.volume.travelAgent(input_file) # makevolume
self.local = Local(self.volume)
self.t_start = datetime.now()
self.service_type = "//*[@id='productline_3']"
self.book_size = "//*[@id='preset_1037_73']"
self.binding = "//*[@id='binding_4']"
self.pg_count = "//*[@id='pagecount']"
self.uid = "//*[@id='loginEmail']" # id/name for logging in
self.pw = "//*[@id='loginPassword']" # password area
self.logbutt = "//*[@id='loginSubmit']" # submit button
self.my_page = "700"
self.next = "//*[@id='fNext']"
self.next_disable = "//*[@id='fNext' and not (@disabled)]"
self.book_title = "//*[@id='title']" # get and put title
self.fname = "//*[@id='firstName']"
self.lname = "//*[@id='lastName']"
self.project_details = "/html/body/div[1]/div[1]/div/div/div[2]"
self.project_id = "/html/body/div[1]/div[1]/div/div/div[2]/div[2]/table/tbody/tr[1]/td[1]"
self.myFiles = "//*[@id='ui-id-2']"
self.cover_upload = "//*[@id='fOnePieceCoverFile']"
self.x_lulu_isbn = "//*[@id='projectInformationSection']/div[2]/table/tbody/tr[1]/td"
self.x_lulu_id = "/html/body/div[1]/div[3]/div[3]/div[2]/div/div[2]/form/div[1]/div[2]/div/div[2]/div[2]/table/tbody/tr[1]/td"
self.x_sell = "/html/body/div[1]/div[3]/div[2]/div[2]/div/div[1]/div/h3/a"
self.sku_x = "/html/body/div[1]/div[2]/div[2]/div[1]/div[2]/div[2]/form/input[1]"
self.bc_image = "/html/body/div/div[3]/div[2]/div[2]/div/div[2]/div[4]/a/img"
self.folders_option = "//*[@id='labelFilter']"
self.num_Xpath = "//*[@id='pageCount']"
self.submit_button = "/html/body/div[1]/div[1]/div/table/tbody/tr/td[2]/div/div[1]/table/tbody/tr/td[1]/input"
self.first_table = "//*[@id='pageLinks']/a[1]"
self.one_file_cover = "/html/body/div[1]/div[5]/div/div[2]/a[2]"
self.c_up_button = "//*[@id='fMegaUpload']"
self.keywords = "//*[@id='keywords']"
self.description = "//*[@id='descr']"
self.copyright = "//*[@id='copyright']"
self.license = "//*[@id='license']"
self.edition = "//*[@id='edition']"
self.publisher = "//*[@id='publisher']"
self.setPrice = "//*[@id='userPrice']"
self.x_lulu_isbn = "//*[@id='projectInformationSection']/div[2]/table/tbody/tr[1]/td"
self.x_sell = "/html/body/div[1]/div[3]/div[2]/div[2]/div/div[1]/div/h3/a"
self.sku_x = "/html/body/div[1]/div[2]/div[2]/div[1]/div[2]/div[2]/form/input[1]"
self.helv_in_enc_file = "/html/body/div[2]/div[1]/div/div[1]/div/div"
profile = FirefoxProfile(
"/Users/wiki/Library/Application Support/Firefox/Profiles/3ig8qcrl.default-1435148476151"
) # profile is necessary to run the greasemonkey scripts.
profile.set_preference("toolkit.startup.max_resumed_crashes", "-1")
self.driver = webdriver.Firefox(profile) # start ff w/profile
self.driver.set_window_position(6920, 0) # browser size and position.
self.driver.set_window_size(220, 220)
print "opening new browser to automate upload of: " + self.volume.title + " on Firefox"
self.driver.get("http://www.lulu.com/author/wizard/index.php?fWizard=hardcover") # navigate to author page
print "cur time= " + str(self.t_start)
self.pdf_pages = self.count_pages(self.local.inFolder + self.volume.round_folder + "/" + input_file)
self.pdf_pages = str(self.pdf_pages)
self.luluCruise() # start!
dir = config["directory_of_videos_to_upload_as_posts_on_timeline"]
interval = config["interval_in_secs"]
likesCount = config["how_many_wall_likes"]
commentsCount = config["how_many_wall_comments"]
reactsCount = config["how_many_wall_reacts"]
postsCount = config["how_many_posts"]
videosCount = config["how_many_videos"]
postsInGroupCount = config["how_many_posts_in_groups"]
token = config["fb_access_token"]
graph = facebook.GraphAPI(access_token=token, version="2.11") #changed from 2.10
picsdir = config["picsdir"]
firefox_profile = config["firefox_profile"]
profile = FirefoxProfile(firefox_profile)
with open("textfile_used_to_find_groups_for_example_from_wikipedia_of_the_city.txt","r") as textwiki:
textwiki = textwiki.read()
text = textwiki.split(' ')
#-------------------------------------------------------------
with open("selectedPosts.json","r") as selectedPosts:
selectedPosts = json.load(selectedPosts)
with open("selectedComments.json","r") as selectedComments:
selectedComments = json.load(selectedComments)
with open("selectedPages.json","r") as selectedPages:
selectedPages = json.load(selectedPages)
def __init__(self, profile):
self.profile = FirefoxProfile(profile)
self.driver = webdriver.Firefox(self.profile)
self.driver.maximize_window()
wait = WebDriverWait(self.driver, 10)
# along the line, Chrome gained the ability to tell whether
# there is touch-enabled hardware on Debian. It is unclear
# what gave it this capability. (It is not based on Chrome's
# version as version 43 used to not detect touch capabilities
# on Debian but later gained the capability. Is it an upgrade
# to Gnome that made the difference??)
#
# Bootstrap does things differently depending on whether touch
# events are available or not. Unfortunately an Xvfb session
# won't report touch to be available even if the host X server
# supports it. So we force it to be able to test for it. Touch
# is becoming mainstream.
#
CHROME_OPTIONS.add_argument("touch-events")
profile = FirefoxProfile()
# profile.set_preference("webdriver.log.file",
# "/tmp/firefox_webdriver.log")
# profile.set_preference("webdriver.firefox.logfile",
# "/tmp/firefox.log")
#
# This turns off the downloading prompt in FF.
#
tmp_path = "selenium_tests/tmp"
shutil.rmtree(tmp_path, True)
os.makedirs(tmp_path)
profile.set_preference("browser.download.folderList", 2)
profile.set_preference("browser.download.manager.showWhenStarting",
False)
profile.set_preference("browser.download.dir", tmp_path)
def get_profile(self):
profile = FirefoxProfile(self.profile_dir)
profile.set_preference('startup.homepage_welcome_url', "about:blank")
profile.set_preference('browser.startup.homepage', "about:blank")
# profile.set_preference('extensions.firebug.netexport.defaultLogDir', self.netlog_dir)
#set socks proxy
profile.set_preference( "network.proxy.type", 1 )
profile.set_preference( "network.proxy.socks_version", 5 )
profile.set_preference( "network.proxy.socks", '127.0.0.1' )
profile.set_preference( "network.proxy.socks_port", 9150)
profile.set_preference( "network.proxy.socks_remote_dns", True )
profile.set_preference( "places.history.enabled", False )
profile.set_preference( "privacy.clearOnShutdown.offlineApps", True )
profile.set_preference( "privacy.clearOnShutdown.passwords", True )
profile.set_preference( "privacy.clearOnShutdown.siteSettings", True )
profile.set_preference( "privacy.sanitize.sanitizeOnShutdown", True )
profile.set_preference( "signon.rememberSignons", False )
profile.set_preference( "network.dns.disablePrefetch", True )
profile.set_preference("extensions.firebug.netexport.pageLoadedTimeout", 10000)
profile.set_preference("extensions.firebug.netexport.showPreview", False)
profile.set_preference("extensions.firebug.netexport.alwaysEnableAutoExport", False)
profile.set_preference("extensions.firebug.DBG_STARTER", False)
profile.set_preference("extensions.firebug.onByDefault", False)
profile.set_preference("extensions.firebug.allPagesActivation", "off")
profile.update_preferences()
return profile
# along the line, Chrome gained the ability to tell whether
# there is touch-enabled hardware on Debian. It is unclear
# what gave it this capability. (It is not based on Chrome's
# version as version 43 used to not detect touch capabilities
# on Debian but later gained the capability. Is it an upgrade
# to Gnome that made the difference??)
#
# Bootstrap does things differently depending on whether touch
# events are available or not. Unfortunately an Xvfb session
# won't report touch to be available even if the host X server
# supports it. So we force it to be able to test for it. Touch
# is becoming mainstream.
#
CHROME_OPTIONS.add_argument("touch-events")
profile = FirefoxProfile()
# profile.set_preference("webdriver.log.file",
# "/tmp/firefox_webdriver.log")
# profile.set_preference("webdriver.firefox.logfile",
# "/tmp/firefox.log")
#
# This turns off the downloading prompt in FF.
#
tmp_path = "selenium_tests/tmp"
shutil.rmtree(tmp_path, True)
os.makedirs(tmp_path)
profile.set_preference("browser.download.folderList", 2)
profile.set_preference("browser.download.manager.showWhenStarting", False)
profile.set_preference("browser.download.dir", tmp_path)
profile.set_preference("browser.helperApps.neverAsk.saveToDisk", "text/xml")
import sqlite3
import time
import re
from selenium import webdriver
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
from selenium.webdriver.firefox.webdriver import FirefoxProfile
profile = FirefoxProfile("C:\\Users\\gokha\\AppData\\Roaming\\Mozilla\\Firefox\\Profiles\\b21rnkyt.kariyer")
profile.add_extension(r'C:\\Users\\gokha\\AppData\\Roaming\\Mozilla\\Firefox\\Profiles\\b21rnkyt.kariyer\\extensions\\adblock.xpi') # for adblockplus
profile.set_preference("extensions.adblockplus.currentVersion","3.7")
binary = FirefoxBinary("C:\\Program Files\\Mozilla Firefox\\firefox.exe")
browser = webdriver.Firefox(firefox_profile=profile, firefox_binary=binary)
browser.maximize_window()
sayfano = 0
linkler = []
durum = "0"
conn = sqlite3.connect('veritaban.db')
c = conn.cursor()
def create_table():
c.execute('CREATE TABLE IF NOT EXISTS basvurulinkleri(urll TEXT,durumu TEXT)')
def giris_yap():
kullanıcı_adı = input()
sifre = input()
browser.get("https://www.kariyer.net/website/kariyerim/login.aspx")
time.sleep(3)
username = browser.find_element_by_xpath("//*[@id='lgnUserName']")
password = browser.find_element_by_xpath("//*[@id='lgnPassword']")
username.send_keys(kullanıcı_adı)
password.send_keys(sifre)
def use_browser(self):
self.use_selenium = True
firefox_profile = FirefoxProfile()
# might as well turn off images since we don't need them
if self.use_proxies:
# if use proxies is true load firefox with proxies
firefox_profile.set_preference("permissions.default.image", 2)
proxy_host, proxy_port = choice(self.proxies).split(':')
firefox_profile.set_preference("network.proxy.type", 1)
firefox_profile.set_preference("network.proxy.http", proxy_host)
firefox_profile.set_preference("network.proxy.http_port", int(proxy_port))
firefox_profile.set_preference("network.proxy.ssl", proxy_host)
firefox_profile.set_preference("network.proxy.ssl_port", int(proxy_port))
self.driver = Firefox(firefox_profile)
self.driver.implicitly_wait(2)
#Import libraries
from selenium import webdriver
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.firefox.webdriver import FirefoxProfile
import time
#Import list user usernames
pathtoinput = '/home/JohnRambo/Flickr_Usernames.csv'
listcontact = open(pathtoinput)
next(listcontact)
#Instantiate a Firefox webbrowser (the profile should be already logged to Flickr)
pathtoprofile = 'path to profile'
profile = FirefoxProfile(pathtoprofile)
browser = webdriver.Firefox(profile)
#Url group invitation
url = "https://www.flickr.com/groups_invite.gne?id=..."
#Loop over your contact list
k = 0
for line in listcontact:
#Url group invitation
browser.get(url)
time.sleep(5)
#Extract Flickr ID
attr = line.rstrip('\n\r').split(';') #Split line
id = (attr[0])
def use_browser(self):
self.use_selenium = True
firefox_profile = FirefoxProfile()
# might as well turn off images since we don't need them
if self.use_proxies:
# if use proxies is true load firefox with proxies
firefox_profile.set_preference("permissions.default.image", 2)
proxy_host, proxy_port = choice(self.proxies).split(':')
firefox_profile.set_preference("network.proxy.type", 1)
firefox_profile.set_preference("network.proxy.http", proxy_host)
firefox_profile.set_preference("network.proxy.http_port",
int(proxy_port))
firefox_profile.set_preference("network.proxy.ssl", proxy_host)
firefox_profile.set_preference("network.proxy.ssl_port",
int(proxy_port))
self.driver = Firefox(firefox_profile)
self.driver.implicitly_wait(2)
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from selenium.webdriver.common.keys import Keys
import os
import time
import json
import shutil
from selenium.webdriver.firefox.webdriver import FirefoxProfile
if __name__ == "__main__":
profile = FirefoxProfile('C:\\Users\\Valdelio\'\\AppData\\Roaming\\mozilla\\Firefox\\Profiles\\a4g555yp.default-release')
path = os.getcwd() + "\\"
with open(path+'data.json', encoding='utf8') as json_file:
data = json.load(json_file)
browser = webdriver.Firefox(profile)
#browser.get(data["Link"])
#browser.find_element_by_xpath("//*[contains(text(), 'Pašalinti' )]").click()
#browser.find_element_by_xpath("//*[contains(text(), 'Patvirtinti ir ištrinti' )]").click()
browser.get('https://new.vinted.lt/items/new')
browser.maximize_window()
#browser.find_element_by_xpath("//button[@id='onetrust-accept-btn-handler']").click()
#Standard Selenium imports
from selenium import webdriver
from selenium.common.exceptions import TimeoutException, NoAlertPresentException, NoSuchElementException
from selenium.webdriver.support.ui import Select, WebDriverWait
#Firefox profile support
from selenium.webdriver.firefox.webdriver import FirefoxProfile
import sys
firefox_profile = r"E:\TEST\Firefox_Profiles\james_conlon_ccr_aws"
url = "http://ccr.hosting.legalaid.technology/ccr/AutoLogin"
print "Starting"
#Access CCR using Firefox profile
ffp_object = FirefoxProfile(firefox_profile)
driver = webdriver.Firefox(ffp_object)
driver.get(url)
#Wait for page
WebDriverWait(driver, 20).until(lambda driver:
"Search For Claims" in driver.page_source
or "<h2>Login Error</h2>" in driver.page_source)
#Search for a case
driver.find_element_by_id("caseNumber").clear()
driver.find_element_by_id("caseNumber").send_keys("T20132011")
driver.find_element_by_xpath("//input[@value='Search']").click()
#Wait for sarch results
WebDriverWait(driver,10).until(lambda driver:
"Search Results" in driver.page_source
or "No claims found" in driver.page_source,driver)
#Crude count of number of results
from selenium import webdriver
from selenium.webdriver.firefox.webdriver import FirefoxProfile
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
profile = FirefoxProfile("/home/patrick/.mozilla/firefox/kn62libz.default")
d = webdriver.Firefox(profile)
d.get("https://jobs.stahl-holding-saar-gruppe.de/bewerber/if_stellenboerse_formapply.php?prj=3181&b=5&oid=1&quelle=0&ie=1")
#------------------------------------------------------------------------------
#` predefined data
#` list_of_my_choices a list of possible choices I always choose
list_of_my_choices = ['herr','1993','ägypten']
#` find all comboboxes and fill them
#` @param driver a driver element
def fillComboboxes(driver):
a = driver.find_elements_by_tag_name('select')
for item in a:
print(getComboboxOptions(item))
chooseCombobox(item)
#------------------------------------------------------------------------------
#` extract options from combobox
#' @param combobox a combobox element
#` @return a list of the options
def getComboboxOptions(combobox):
o = combobox.find_elements_by_tag_name("option")
i = []
for item in o:
i.append(item.text)
CHROME_OPTIONS = Options()
#
# This prevents getting message shown in Chrome about
# --ignore-certificate-errors
#
# --test-type is an **experimental** option. Reevaluate this
# --use.
#
CHROME_OPTIONS.add_argument("test-type")
# Tried these to see if they were stopping a slowdown with Chrome,
# did not work.
# CHROME_OPTIONS.add_argument('--dns-prefetch-disable')
# CHROME_OPTIONS.add_argument('--no-proxy-server')
profile = FirefoxProfile()
# profile.set_preference("webdriver.log.file",
# "/tmp/firefox_webdriver.log")
# profile.set_preference("webdriver.firefox.logfile",
# "/tmp/firefox.log")
FIREFOX_PROFILE = profile
CHROMEDRIVER_PATH = "chromedriver"
SC_TUNNEL_PATH = "sc"
BSLOCAL_PATH = "BrowserStackLocal"
if CONFIG.remote and not REMOTE_SERVICE:
raise ValueError("you must pass a service argument to behave")
# May be required to get native events.
# FIREFOX_BINARY = FirefoxBinary("/home/ldd/src/firefox-24/firefox")
def _set_profile():
useragent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) ' \
'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36'
profile = FirefoxProfile()
profile.set_preference('general.useragent.override', useragent)
return profile
def get_profile(self):
profile = FirefoxProfile(self.profile_dir)
profile.set_preference('startup.homepage_welcome_url', "about:blank")
profile.set_preference('browser.startup.homepage', "about:blank")
# profile.set_preference('extensions.firebug.netexport.defaultLogDir', self.netlog_dir)
#set socks proxy
profile.set_preference("network.proxy.type", 1)
profile.set_preference("network.proxy.socks_version", 5)
profile.set_preference("network.proxy.socks", '127.0.0.1')
profile.set_preference("network.proxy.socks_port", 9150)
profile.set_preference("network.proxy.socks_remote_dns", True)
profile.set_preference("places.history.enabled", False)
profile.set_preference("privacy.clearOnShutdown.offlineApps", True)
profile.set_preference("privacy.clearOnShutdown.passwords", True)
profile.set_preference("privacy.clearOnShutdown.siteSettings", True)
profile.set_preference("privacy.sanitize.sanitizeOnShutdown", True)
profile.set_preference("signon.rememberSignons", False)
profile.set_preference("network.dns.disablePrefetch", True)
profile.set_preference(
"extensions.firebug.netexport.pageLoadedTimeout", 10000)
profile.set_preference("extensions.firebug.netexport.showPreview",
False)
profile.set_preference(
"extensions.firebug.netexport.alwaysEnableAutoExport", False)
profile.set_preference("extensions.firebug.DBG_STARTER", False)
profile.set_preference("extensions.firebug.onByDefault", False)
profile.set_preference("extensions.firebug.allPagesActivation", "off")
profile.update_preferences()
return profile
def login(self, url, username="", password="", firefox_profile=""):
"""
Login to CCR either via Portal or Modify Headers(using Firefox profile)
Args:
url - url (either portal or /ccr/Autlogin for Modify headers)
username - portal username (not needed if Modify headers being used)
passwored - portal password (not needed if Modify headers being used)
firefox_profile - path to firefox profile (only needed if Modify headers being used)
# *NEEDS ADAPTING FOR NEW PORTAL*
Returns:
True if login succesfull, otherwise False
"""
#Create webdriver instance.
#Diferent settings depending on whether we're accessing via Modify Headers
if firefox_profile:
#Use specified Firefox Profile (Modify Headers)
ffp_object = FirefoxProfile(firefox_profile)
self.driver = webdriver.Firefox(ffp_object)
else:
#Normal default Firefox profile (portal login)
self.driver = webdriver.Firefox()
driver = self.driver
result = False #flag will be changed to True if login successful
if firefox_profile:
#Direct login (modify headers)
driver.get(url)
else:
#Normal Portal Login (old portal)
driver.get(url)
driver.find_element_by_id("userField").send_keys(username)
driver.find_element_by_id("passField").send_keys(password)
driver.find_element_by_name("submit").click()
#Wait a bit for response
try:
WebDriverWait(self.driver, 10).until(
lambda driver: ">Logged in as:" in driver.page_source or
">Authentication failed" in driver.page_source or
"This page can?t be displayed" in driver.page_source or
"Secure Connection Failed" in driver.page_source or
"HTTP Status 404" in driver.page_source)
except TimeoutException:
pass
#Stop is login was not successful
if "Logged in as:" not in driver.page_source:
return False
##print [e.text for e in driver.find_elements_by_tag_name("a")]
#Click portal link
link = u'Crown Court Remuneration (CCR)'
#Set new_win to be "y" if link's target attribute is 'LSCAppPopUp'
new_win = False
if driver.find_element_by_link_text(link).get_attribute(
"target") == 'LSCAppPopUp':
new_win = True
driver.find_element_by_link_text(link).click()
# Need to switch to new window
if new_win:
##WebDriverWait(driver,20).until(lambda x: len(x.window_handles)==2,self.driver)
WebDriverWait(driver,
20).until(lambda x: len(x.window_handles) == 2)
newwindow = driver.window_handles[-1]
driver.switch_to_window(newwindow)
#Wait for expected CCR page
WebDriverWait(
self.driver,
20).until(lambda driver: "Search For Claims" in driver.page_source
or "<h2>Login Error</h2>" in driver.page_source)
#Set success flag if login successful
if "Search For Claims" in driver.page_source:
result = True
return result
def setup_test_environment(self, **kwargs):
super(SeleniumTestSuiteRunner, self).setup_test_environment(**kwargs)
profile = FirefoxProfile()
profile.add_extension(settings.FIREBUG_EXTENSION)
profile.add_extension(settings.FIRESTARTER_EXTENSION)
profile.set_preference('extensions.firebug.currentVersion', '1.10.3')
profile.set_preference('extensions.firebug.allPagesActivation', 'on')
profile.set_preference('extensions.firebug.previousPlacement', 1)
profile.set_preference('extensions.firebug.onByDefault', True)
profile.set_preference('extensions.firebug.showFirstRunPage', False)
profile.set_preference('extensions.firebug.defaultPanelName', 'console')
profile.set_preference('extensions.firebug.net.enableSites', True)
profile.set_preference('extensions.firebug.console.enableSites', True)
self.__class__.selenium = WebDriver(firefox_profile=profile)