def __init__(self, path=None, arabic=True):
self.arabic = arabic
# Browsing and writing managers
self.br = Browser()
self.wr = Writer(path) if path else Writer()
# An array for scrapped books
self._books_ids = []
def setupDock(self):
# 建立输出运行结果的窗口
self.rightBrowser = Browser() # 建立在窗口右端
self.rightDock = QDockWidget('运行结果/内存', self)
self.rightDock.setFeatures(QDockWidget.DockWidgetClosable)
self.rightDock.setWidget(self.rightBrowser)
self.addDockWidget(Qt.RightDockWidgetArea, self.rightDock)
self.rightDock.hide() # 窗口自动隐藏
# 建立输出调试过程的窗口
self.bottomBrowser = Browser() # 建立在窗口底端
self.bottomDock = QDockWidget('调试窗口', self)
self.bottomDock.setFeatures(QDockWidget.DockWidgetClosable)
self.bottomDock.setWidget(self.bottomBrowser)
self.addDockWidget(Qt.BottomDockWidgetArea, self.bottomDock)
self.bottomDock.hide()
def createBrowser(self):
self.listWidget.clear()
if ((self.keywordsName != '') & (self.docName != '')):
self.flag = 1
print(self.checkBox.isChecked())
self.browser = Browser(self.keywordsName, self.docName,
self.checkBox.isChecked())
def setUp(self):
self.driver = Browser().getbrowser("chrome")
self.driver.get(self.url.webViewerUAT)
self.driverUtility = DriverUtility(self.driver, self.log)
self.loginPageObject = LoginPageObject(self.driverUtility, self.log)
yield
self.driver.close()
def crawl_nexon_board():
raw_data = []
browser = Browser()
browser.get_page(URL.BASE_URL)
browser.click_element_by_xpath(ElementXpath.POST_TITLES_ON_BOARD)
recent_post_id = int(
re.search('n4articlesn=(\d+)', browser.get_current_url()).group(1))
for i in range(0, 10):
url = PathUtils.fromTemplate(URL.POST_URL, str(recent_post_id - i))
browser.get_page(url)
title = browser.get_element_by_xpath(
ElementXpath.TITLE_ON_POST).text
content = browser.get_element_by_xpath(
ElementXpath.CONTENT_ON_POST).get_attribute("content")
comments = list(
map(
lambda element: element.text,
browser.get_elements_by_xpath(
ElementXpath.COMMENTS_ON_POST)))
raw_data.append(title)
raw_data.append(content)
raw_data.extend(comments)
return raw_data
def __init__(self, *args, obj=None, **kwargs):
super().__init__(*args, **kwargs)
self.browser = Browser()
self.setupUi(self)
self.load_config()
self.set_ui()
self.set_signals()
def __init__(self) -> None:
self.offices = []
self.agents = []
self.agentNames = []
self.agentPositions = []
self.agentNumbers_1 = []
self.agentNumbers_2 = []
self.browser = Browser()
def create_widgets(self):
self.browser = Browser()
self.create_butonn_1()
self.create_button_2()
self.layout.addWidget(self.browser, 1, 1)
self.layout.addWidget(self.button1, 2, 1)
self.layout.addWidget(self.button2, 2, 2)
def __init__(self) -> None:
self.officeNames = []
self.officeAddress = []
self.officeProvinces = []
self.phoneNumbers = []
self.websiteLinks = []
self.emails = []
self.agents = []
self.browser = Browser()
def add_new_tab(self, qurl, label):
# Instead of browser it should be called WebView !
browser = Browser()
browser.urlChanged.connect(
lambda qurl, browser=browser: self.update_url_bar(qurl, browser))
indexTab = self.tabs.addTab(browser, label)
# We need to update the url !
self.tabs.currentWidget().setUrl(QUrl(qurl))
return indexTab
def getRequest(self, pluginName, cj=None):
req = Browser(self.bucket, self.getOptions())
if cj:
req.setCookieJar(cj)
else:
req.setCookieJar(CookieJar(pluginName))
return req
def __init__(self, path=None, lang="ar", edition_reviews=False):
# Language of reviews to be scraped
self._lang = lang
# Instantiate browsing and writing managers
self.wr = Writer(path) if path else Writer()
self.br = Browser(edition_reviews)
# Initialize an empty threads list
self._threads = []
# Counter for reviews from different languages
self._invalid = None
def getRequest(self, pluginName, cj=None):
# TODO: mostly obsolete, browser could be removed completely
req = Browser(self.bucket, self.getOptions())
if cj:
req.setCookieJar(cj)
else:
req.setCookieJar(CookieJar(pluginName))
return req
def __init__(self, baselink, username=None, password=None):
self.browser = Browser()
self.baselink = str(baselink).strip("/") + "/"
self.browser.open_page(self.baselink + 'login.aspx')
try:
if username == None:
self.browser.sendClick("//input[@id='defaultuser']")
else:
self.__login(username, password)
except:
pass
def follow_new(self, event):
url, target = self.whichurl(event)
if url:
app = self.context.app
self['background'] = app.prefs.Get(STYLEGROUP, ATEMPPREF)
url = self.context.get_baseurl(url)
from Browser import Browser
Browser(app.root, app).context.load(url)
self['background'] = app.prefs.Get(STYLEGROUP, AHISTPREF)
else:
self.context.viewer.leave_message()
def anchor_click_new(self, event):
here = self.text.index("@%d,%d" % (event.x, event.y))
if self.current_index != here:
return
url = self.find_tag_url()
if url:
url, target = self.split_target(url)
self.master.update_idletasks()
from Browser import Browser
app = self.context.app
b = Browser(app.root, app)
b.context.load(self.context.get_baseurl(url))
self.remove_temp_tag(histify=1)
def directUpload(self, filename, private=False, test=False):
if self.bot is None:
self.bot = Browser().getBot()
self.webbot = Bot(self.bot)
self.bot.get(self.url)
utils.randomTimeQuery()
self.cookies = Cookies(self.bot)
self.bot.refresh()
try:
file_input_element = self.webbot.getVideoUploadInput()
except Exception as e:
print(f"Error: {e}")
print(
"Major error, cannot find the file upload button, please update getVideoUploadInput() in Bot.py"
)
file_input_element = None
exit()
abs_path = os.path.join(os.getcwd(), filename)
try:
file_input_element.send_keys(abs_path)
except StaleElementReferenceException as e:
try:
self.bot.implicitly_wait(5)
file_input_element = self.webbot.getVideoUploadInput()
file_input_element.send_keys(abs_path)
except Exception as e:
print(
"Major error, cannot find the file upload button, please update getVideoUploadInput() in Bot.py"
)
exit()
# We need to wait until it is uploaded and then clear input.
self.addCaptions()
utils.randomTimeQuery()
if private:
self.webbot.selectPrivateRadio() # private video selection
utils.randomTimeQuery()
else:
"""
self.webbot.selectPublicRadio() # public video selection
utils.randomTimeQuery()
"""
pass
if not test:
self.webbot.uploadButtonClick() # upload button
input("Press any button to exit")
def start_checking():
browser = Browser()
clear()
count = 0
reposition(0, 0)
print(colorama.Fore.BLUE + INTRO + colorama.Fore.RESET)
valid = False
reset_screen()
try:
count = 0
while not valid:
website, line = website_list.pop(0)
website_list.append([website, line])
curtime = datetime.now()
seconds_passed = int(
(curtime - website.last_checked).total_seconds())
if seconds_passed < REFRESH_PERIOD:
time.sleep(REFRESH_PERIOD - seconds_passed)
else:
count += 1
cur_price, in_stock, name, item_valid = browser.check_website(
website)
website.update_website(cur_price, in_stock, name)
if item_valid:
print(colorama.Fore.GREEN + str(website) +
colorama.Fore.RESET)
time.sleep(3)
valid = True
else:
if count == len(website_list) * 3:
reset_screen()
count = 0
reposition(line, 0)
print(colorama.Fore.RED + str(website) +
colorama.Fore.RESET)
except KeyboardInterrupt:
try:
browser.driver.close()
time.sleep(1)
except Exception as e:
print(e)
def getRequest(self, pluginName, account=None, type="HTTP"):
self.lock.acquire()
if type == "XDCC":
return XDCCRequest(proxies=self.getProxies())
req = Browser(self.bucket, self.getOptions())
if account:
cj = self.getCookieJar(pluginName, account)
req.setCookieJar(cj)
else:
req.setCookieJar(CookieJar(pluginName))
self.lock.release()
return req
def add_website(url, site_type, max_price):
browser = Browser()
if url not in URL_MAP.map.values():
uuid = URL_MAP.add(url)
website = Website(uuid,
url=url,
site_type=site_type,
max_price=max_price)
cur_price, in_stock, name, value = browser.check_website(website)
website.update_website(cur_price, in_stock, name)
created = True
else:
created = False
browser.driver.close()
return created
def _get(self, url, sub_dir):
capture = Capture(sub_dir, self.conf)
browser = Browser(url, sub_dir, self.conf)
links = []
try:
capture.run()
sleep(3)
html = browser.get()
links = self._get_links(html)
sleep(30)
except WebDriverException:
self._create_exception_file(traceback.format_exc(), sub_dir)
except KeyboardInterrupt:
self._create_exception_file(traceback.format_exc(), sub_dir)
finally:
capture.kill()
browser.close()
return links
def main():
# First, we are going to create a browser
google_chrome_driver_path = '/home/hamid/Desktop/javad/research/VENV/JTest/chromedriver'
google_chrome_object = Browser(google_chrome_driver_path)
google_chrome = google_chrome_object.create_chrome_tab()
# Then we are going to start our browser to crawl the webpage
url_for_scrapping = 'https://marketplace.atlassian.com/search?category=Time%20tracking&product=jira'
google_chrome.get(url_for_scrapping)
# After that we are using the wait().until to make us sure that the webpage loads successfully
try:
wait_time = 5000
element = WebDriverWait(google_chrome, wait_time).until(
EC.presence_of_element_located(
(By.XPATH,
"//h3[@class='sc-ghsgMZ giyMKw sc-kkbgRg hkWXqv']")))
javad_spider = Jscrapper(google_chrome)
# Calculate the number of click on "More Results" button
number_per_page = 24
search_result_count = javad_spider.search_result_count_finder()
number_for_click_on_button = (search_result_count) // (number_per_page)
for num in range(number_for_click_on_button):
# Click on button for seeing all plugins
javad_spider.more_result()
# wait for loading the page successfully
javad_spider.wait_for_loading()
# Create CSV file
javad_spider.csv_maker()
# Start collecting all the plugins information
plugins = javad_spider.plugin_finder()
javad_spider.fetch_data(plugins)
finally:
google_chrome.quit()
def uploadVideo(self,
video_dir,
videoText,
startTime=0,
endTime=0,
private=True,
test=False,
scheduled=False,
schdate="",
schtime=""):
video_dir = self.downloadIfYoutubeURL(video_dir)
if not video_dir:
return
if self.bot is None:
self.bot = Browser().getBot()
self.webbot = Bot(self.bot)
self.userRequest["dir"] = video_dir
self.checkFileExtensionValid()
self.userRequest["cap"] = self.IO.getHashTagsFromFile()
# Initiate bot if isn't already.
self.bot.get(self.url)
self.userRequest["vidTxt"] = videoText
# Cookies loaded here.
self.cookies = Cookies(self.bot)
self.bot.refresh()
# User now has logged on and can upload videos
time.sleep(3)
self.inputVideo(startTime, endTime)
self.addCaptions()
utils.randomTimeQuery()
if private:
self.webbot.selectPrivateRadio() # private video selection
else:
self.webbot.selectPublicRadio() # public video selection
utils.randomTimeQuery()
if not test:
self.webbot.uploadButtonClick() # upload button
input("Press any button to exit")
def add_new_tab(self, qurl, label):
# Instead of browser it should be called WebView !
browser = Browser()
browser.urlChanged.connect(
lambda qurl, browser=browser: self.update_url_bar(qurl, browser))
indexTab = self.tabs.addTab(browser, label)
self.tabs.setCurrentIndex(indexTab)
# We need to update the url !
if qurl.startswith('zero://'):
# ZeroNet protocol
url_array = qurl.split('/')
qurl = 'http://127.0.0.1:43110/' + url_array[2]
elif qurl.startswith('http://'):
# http protocol
pass
else:
# Nothing mentionned
qurl = 'http://127.0.0.1:43110/' + qurl
self.tabs.currentWidget().setUrl(QUrl(qurl))
return indexTab
def getRequest(self, pluginName, account=None, type="HTTP", **kwargs):
self.lock.acquire()
options = self.getOptions()
options.update(kwargs) # submit kwargs as additional options
if type == "XDCC":
req = XDCCRequest(self.bucket, options)
else:
req = Browser(self.bucket, options)
if account:
cj = self.getCookieJar(pluginName, account)
req.setCookieJar(cj)
else:
req.setCookieJar(CookieJar(pluginName))
self.lock.release()
return req
def setup(request, setUpClass):
print("initiating chrome driverd")
driver = Browser().getbrowser("chrome")
url = URL()
driver.get(url.webViewerUAT)
utility = Utility()
# utility.createLogFolder()
log = open(utility.logpath + "/WV-00.txt", "a+")
driverUtility = DriverUtility(driver, log)
loginPageObject = LoginPageObject(driverUtility, log)
request.cls.driver = driver
request.cls.url1 = url
request.cls.utility = utility
request.cls.driverUtility = driverUtility
request.cls.loginPageObject = loginPageObject
print("setup ended")
yield driver
driver.close()
def crawl(self, **args):
source = args['source']
ext = args['ext']
reply_time = args['reply_time']
br = Browser()
page = br.open(self.baseurl)
new_reply_time = reply_time
while True:
links = PageParser.parse(page, source)
for i, link in enumerate(links):
if reply_time < link.reply_time:
if i is 0:
new_reply_time = link.reply_time
self.db.insert('\t'.join(
[str(link), source, json.dumps(ext)]))
else:
return new_reply_time
try:
page = br.follow_link(text='后页>')
except:
Logger.info('finished!')
break
return new_reply_time
def lib():
return Browser()
def test_none(lib: Browser):
assert lib.get_timeout(None) == 10000
assert Browser(timeout=timedelta(seconds=0.001)).timeout == 1
"""
Method to save screenshot to a given path with the function name
:param funcName: Function name
:param defaultPath: is an optional paramter
:return: None
"""
path = os.path.dirname(os.path.dirname(__file__))
path = path + "\\reports\\report_" + str(
datetime.datetime.now().date()) + os.sep
try:
if not os.path.exists(path):
print("path is not there creating...")
os.makedirs(path)
name = path + funcName + str(
datetime.datetime.now().time()).replace(":", "_") + ".jpg"
print(name)
self._browser.get_screenshot_as_file(name)
except Exception, e:
print(e)
if __name__ == "__main__":
params = {"name": "Vinay"}
myBrowser = Browser(params)
myBrowser.go_to("http://google.com")
Webaction = WebElementAction(myBrowser)
Webaction.input_text("SearchButton", "Vinay")
Webaction.submit_form("SearchButton")
#Webaction.press_key("SearchButton","ENTER")
time.sleep(3)
myBrowser.quit()
