def __init__(self, path=None, arabic=True):
self.arabic = arabic
# Browsing and writing managers
self.br = Browser()
self.wr = Writer(path) if path else Writer()
# An array for scrapped books
self._books_ids = []
def setUp(self):
self.driver = Browser().getbrowser("chrome")
self.driver.get(self.url.webViewerUAT)
self.driverUtility = DriverUtility(self.driver, self.log)
self.loginPageObject = LoginPageObject(self.driverUtility, self.log)
yield
self.driver.close()
def __init__(self) -> None:
self.offices = []
self.agents = []
self.agentNames = []
self.agentPositions = []
self.agentNumbers_1 = []
self.agentNumbers_2 = []
self.browser = Browser()
def __init__(self, *args, obj=None, **kwargs):
super().__init__(*args, **kwargs)
self.browser = Browser()
self.setupUi(self)
self.load_config()
self.set_ui()
self.set_signals()
def test_no_keyword(browser: Browser.Browser):
keyword = browser._parse_run_on_failure_keyword("nOnE")
assert keyword.name is None
assert keyword.args == tuple()
keyword = browser._parse_run_on_failure_keyword(None)
assert keyword.name is None
assert keyword.args == tuple()
assert keyword.kwargs == {}
def __init__( self, *args, **kwargs ):
Browser.__init__( self, *args, **kwargs )
self.curList = [] # Current list of item/category
#self.racineDisplayList = [ Item.TYPE_SCRIPT, Item.TYPE_PLUGIN ]
self.racineDisplayList = [ Item.TYPE_NEW, Item.TYPE_SCRIPT_CAT, Item.TYPE_PLUGIN ]
# Create History instance in order to store browsing history
self.history = History()
def getRequest(self, pluginName, cj=None):
req = Browser(self.bucket, self.getOptions())
if cj:
req.setCookieJar(cj)
else:
req.setCookieJar(CookieJar(pluginName))
return req
def __init__(self) -> None:
self.officeNames = []
self.officeAddress = []
self.officeProvinces = []
self.phoneNumbers = []
self.websiteLinks = []
self.emails = []
self.agents = []
self.browser = Browser()
def test_external_keyword(browser: Browser.Browser):
keyword = browser._parse_run_on_failure_keyword("Kala")
assert keyword.name == "Kala"
assert keyword.original_name == "Kala"
assert keyword.args == tuple()
keyword = browser._parse_run_on_failure_keyword("Kala arg1 arg2")
assert keyword.name == "Kala"
assert keyword.original_name == "Kala"
assert keyword.args == ("arg1", "arg2")
assert keyword.kwargs == {}
def __init__(self, path=None, lang="ar", edition_reviews=False):
# Language of reviews to be scraped
self._lang = lang
# Instantiate browsing and writing managers
self.wr = Writer(path) if path else Writer()
self.br = Browser(edition_reviews)
# Initialize an empty threads list
self._threads = []
# Counter for reviews from different languages
self._invalid = None
def __init__(self, baselink, username=None, password=None):
self.browser = Browser()
self.baselink = str(baselink).strip("/") + "/"
self.browser.open_page(self.baselink + 'login.aspx')
try:
if username == None:
self.browser.sendClick("//input[@id='defaultuser']")
else:
self.__login(username, password)
except:
pass
def test_library_keyword(browser: Browser.Browser):
keyword = browser._parse_run_on_failure_keyword("Take Screenshot")
assert keyword.name == "take_screenshot"
assert keyword.original_name == "Take Screenshot"
assert keyword.args is tuple()
assert keyword.kwargs == {}
keyword = browser._parse_run_on_failure_keyword(
"tAke sCreenshot filename.png //button True")
assert keyword.name == "take_screenshot"
assert keyword.original_name == "tAke sCreenshot"
assert keyword.args == ("filename.png", "//button", True)
assert keyword.kwargs == {}
def __init__(self, *args, **kwargs):
Browser.__init__(self, *args, **kwargs)
# Creation du configCtrl
from CONF import configCtrl
self.configManager = configCtrl()
if not self.configManager.is_conf_valid: raise
self.host = self.configManager.host
self.user = self.configManager.user
self.password = self.configManager.password
self.remotedirList = self.configManager.remotedirList
#self.localdirList = self.configManager.localdirList
#self.downloadTypeList = self.configManager.downloadTypeLst
self.racineDisplayList = [
Item.TYPE_SKIN, Item.TYPE_SCRAPER, Item.TYPE_SCRIPT,
Item.TYPE_PLUGIN
]
self.pluginDisplayList = [
Item.TYPE_PLUGIN_MUSIC, Item.TYPE_PLUGIN_PICTURES,
Item.TYPE_PLUGIN_PROGRAMS, Item.TYPE_PLUGIN_VIDEO
]
self.scraperDisplayList = [
Item.TYPE_SCRAPER_MUSIC, Item.TYPE_SCRAPER_VIDEO
]
self.connected = False # status de la connection ( inutile pour le moment )
self.index = ""
self.rightstest = ""
self.CACHEDIR = self.configManager.CACHEDIR
# Connection to FTP server
try:
self.passionFTPCtrl = FtpDownloadCtrl(self.host, self.user,
self.password,
self.remotedirList,
self.CACHEDIR)
self.connected = True
except:
print "PassionFtpBrowser::__init__: Exception durant la connection FTP"
print "Impossible de se connecter au serveur FTP: %s" % self.host
print_exc()
# Creons ItemInfosManager afin de recuperer les descriptions des items
self.itemInfosManager = ItemInfosManager(mainwin=self)
self.infoswarehouse = self.itemInfosManager.get_info_warehouse()
def directUpload(self, filename, private=False, test=False):
if self.bot is None:
self.bot = Browser().getBot()
self.webbot = Bot(self.bot)
self.bot.get(self.url)
utils.randomTimeQuery()
self.cookies = Cookies(self.bot)
self.bot.refresh()
try:
file_input_element = self.webbot.getVideoUploadInput()
except Exception as e:
print(f"Error: {e}")
print(
"Major error, cannot find the file upload button, please update getVideoUploadInput() in Bot.py"
)
file_input_element = None
exit()
abs_path = os.path.join(os.getcwd(), filename)
try:
file_input_element.send_keys(abs_path)
except StaleElementReferenceException as e:
try:
self.bot.implicitly_wait(5)
file_input_element = self.webbot.getVideoUploadInput()
file_input_element.send_keys(abs_path)
except Exception as e:
print(
"Major error, cannot find the file upload button, please update getVideoUploadInput() in Bot.py"
)
exit()
# We need to wait until it is uploaded and then clear input.
self.addCaptions()
utils.randomTimeQuery()
if private:
self.webbot.selectPrivateRadio() # private video selection
utils.randomTimeQuery()
else:
"""
self.webbot.selectPublicRadio() # public video selection
utils.randomTimeQuery()
"""
pass
if not test:
self.webbot.uploadButtonClick() # upload button
input("Press any button to exit")
def start_checking():
browser = Browser()
clear()
count = 0
reposition(0, 0)
print(colorama.Fore.BLUE + INTRO + colorama.Fore.RESET)
valid = False
reset_screen()
try:
count = 0
while not valid:
website, line = website_list.pop(0)
website_list.append([website, line])
curtime = datetime.now()
seconds_passed = int(
(curtime - website.last_checked).total_seconds())
if seconds_passed < REFRESH_PERIOD:
time.sleep(REFRESH_PERIOD - seconds_passed)
else:
count += 1
cur_price, in_stock, name, item_valid = browser.check_website(
website)
website.update_website(cur_price, in_stock, name)
if item_valid:
print(colorama.Fore.GREEN + str(website) +
colorama.Fore.RESET)
time.sleep(3)
valid = True
else:
if count == len(website_list) * 3:
reset_screen()
count = 0
reposition(line, 0)
print(colorama.Fore.RED + str(website) +
colorama.Fore.RESET)
except KeyboardInterrupt:
try:
browser.driver.close()
time.sleep(1)
except Exception as e:
print(e)
def getRequest(self, pluginName, account=None, type="HTTP"):
self.lock.acquire()
if type == "XDCC":
return XDCCRequest(proxies=self.getProxies())
req = Browser(self.bucket, self.getOptions())
if account:
cj = self.getCookieJar(pluginName, account)
req.setCookieJar(cj)
else:
req.setCookieJar(CookieJar(pluginName))
self.lock.release()
return req
class FarooSearch(object):
SEARCH_URL = "http://www.faroo.com/api?q=%(query)s&start=%(start)s&length=%(length)s&l=en&src=web&f=json"
def __init__(self, query, start, length):
self.query = query.replace(' ', '%20')
self.start = start
self.length = length
self.browser = Browser()
def _get_results(self):
url = FarooSearch.SEARCH_URL
actual_url = url % {'query': self.query,
'start': self.start,
'length': self.length}
try:
page = self.browser.get_page(actual_url)
except BrowserError:
raise SearchError ("Failed getting %s: %s") % (e.url, e.error)
#return (unicodedata.normalize('NFKD', page).encode('ascii','ignore')).decode("utf-8")
return page.decode("utf-8")
def setupDock(self):
# 建立输出运行结果的窗口
self.rightBrowser = Browser() # 建立在窗口右端
self.rightDock = QDockWidget('运行结果/内存', self)
self.rightDock.setFeatures(QDockWidget.DockWidgetClosable)
self.rightDock.setWidget(self.rightBrowser)
self.addDockWidget(Qt.RightDockWidgetArea, self.rightDock)
self.rightDock.hide() # 窗口自动隐藏
# 建立输出调试过程的窗口
self.bottomBrowser = Browser() # 建立在窗口底端
self.bottomDock = QDockWidget('调试窗口', self)
self.bottomDock.setFeatures(QDockWidget.DockWidgetClosable)
self.bottomDock.setWidget(self.bottomBrowser)
self.addDockWidget(Qt.BottomDockWidgetArea, self.bottomDock)
self.bottomDock.hide()
def createBrowser(self):
self.listWidget.clear()
if ((self.keywordsName != '') & (self.docName != '')):
self.flag = 1
print(self.checkBox.isChecked())
self.browser = Browser(self.keywordsName, self.docName,
self.checkBox.isChecked())
def __init__(self, *args, **kwargs):
Browser.__init__(self, *args, **kwargs)
# Creation du configCtrl
from CONF import configCtrl
self.configManager = configCtrl()
if not self.configManager.is_conf_valid:
raise
self.host = self.configManager.host
self.user = self.configManager.user
self.password = self.configManager.password
self.remotedirList = self.configManager.remotedirList
# self.localdirList = self.configManager.localdirList
# self.downloadTypeList = self.configManager.downloadTypeLst
self.racineDisplayList = [Item.TYPE_SKIN, Item.TYPE_SCRAPER, Item.TYPE_SCRIPT, Item.TYPE_PLUGIN]
self.pluginDisplayList = [
Item.TYPE_PLUGIN_MUSIC,
Item.TYPE_PLUGIN_PICTURES,
Item.TYPE_PLUGIN_PROGRAMS,
Item.TYPE_PLUGIN_VIDEO,
]
self.scraperDisplayList = [Item.TYPE_SCRAPER_MUSIC, Item.TYPE_SCRAPER_VIDEO]
self.connected = False # status de la connection ( inutile pour le moment )
self.index = ""
self.rightstest = ""
self.CACHEDIR = self.configManager.CACHEDIR
# Connection to FTP server
try:
self.passionFTPCtrl = FtpDownloadCtrl(
self.host, self.user, self.password, self.remotedirList, self.CACHEDIR
)
self.connected = True
except:
print "PassionFtpBrowser::__init__: Exception durant la connection FTP"
print "Impossible de se connecter au serveur FTP: %s" % self.host
print_exc()
# Creons ItemInfosManager afin de recuperer les descriptions des items
self.itemInfosManager = ItemInfosManager(mainwin=self)
self.infoswarehouse = self.itemInfosManager.get_info_warehouse()
def create_widgets(self):
self.browser = Browser()
self.create_butonn_1()
self.create_button_2()
self.layout.addWidget(self.browser, 1, 1)
self.layout.addWidget(self.button1, 2, 1)
self.layout.addWidget(self.button2, 2, 2)
def main():
username = file('username.txt').read()
password = file('password.txt').read()
if not os.path.exists('output'):
os.mkdir('output')
web = Browser()
# login
web.visit('https://www.netflix.com/Login')
body = {
'nextpage':'https://www.netflix.com/Default',
'email':username,
'movieid':'',
'trkid':'',
'password1':password,
'SubmitButton':'Click Here to Continue',
}
web.visit('https://www.netflix.com/Login','POST',body)
web.visit('https://www.netflix.com/Queue')
# update queue
#q = pickle.load(file('queuepickle.txt','r'))
q = NetflixQueue()
q.parsehtml(web.history[-1][0])
tosort = {}
for i in range(len(q.movies)-1):
tosort[i] = q.movies[i].recrating
sortorder = vlib.vdict.vsort(tosort)
sortorder.reverse()
body = {
'updateQueueBtn':'Update Your Queue'
}
# fields : or123456=i, op123456=i, r123456 (checkbox) [for delete?]
# OP is previous qpos, OR is next
j = 0
for i in sortorder:
j = j + 1
body['OP'+str(q.movies[i].id)] = q.movies[i].qpos
body['OR'+str(q.movies[i].id)] = j
print '#'+str(j)
print '%d. %s [%r]' % (q.movies[i].qpos,q.movies[i].title,q.movies[i].recrating)
web.visit('http://www.netflix.com/Queue','POST',body)
def add_website(url, site_type, max_price):
browser = Browser()
if url not in URL_MAP.map.values():
uuid = URL_MAP.add(url)
website = Website(uuid,
url=url,
site_type=site_type,
max_price=max_price)
cur_price, in_stock, name, value = browser.check_website(website)
website.update_website(cur_price, in_stock, name)
created = True
else:
created = False
browser.driver.close()
return created
def add_new_tab(self, qurl, label):
# Instead of browser it should be called WebView !
browser = Browser()
browser.urlChanged.connect(
lambda qurl, browser=browser: self.update_url_bar(qurl, browser))
indexTab = self.tabs.addTab(browser, label)
# We need to update the url !
self.tabs.currentWidget().setUrl(QUrl(qurl))
return indexTab
def _get(self, url, sub_dir):
capture = Capture(sub_dir, self.conf)
browser = Browser(url, sub_dir, self.conf)
links = []
try:
capture.run()
sleep(3)
html = browser.get()
links = self._get_links(html)
sleep(30)
except WebDriverException:
self._create_exception_file(traceback.format_exc(), sub_dir)
except KeyboardInterrupt:
self._create_exception_file(traceback.format_exc(), sub_dir)
finally:
capture.kill()
browser.close()
return links
def main():
# First, we are going to create a browser
google_chrome_driver_path = '/home/hamid/Desktop/javad/research/VENV/JTest/chromedriver'
google_chrome_object = Browser(google_chrome_driver_path)
google_chrome = google_chrome_object.create_chrome_tab()
# Then we are going to start our browser to crawl the webpage
url_for_scrapping = 'https://marketplace.atlassian.com/search?category=Time%20tracking&product=jira'
google_chrome.get(url_for_scrapping)
# After that we are using the wait().until to make us sure that the webpage loads successfully
try:
wait_time = 5000
element = WebDriverWait(google_chrome, wait_time).until(
EC.presence_of_element_located(
(By.XPATH,
"//h3[@class='sc-ghsgMZ giyMKw sc-kkbgRg hkWXqv']")))
javad_spider = Jscrapper(google_chrome)
# Calculate the number of click on "More Results" button
number_per_page = 24
search_result_count = javad_spider.search_result_count_finder()
number_for_click_on_button = (search_result_count) // (number_per_page)
for num in range(number_for_click_on_button):
# Click on button for seeing all plugins
javad_spider.more_result()
# wait for loading the page successfully
javad_spider.wait_for_loading()
# Create CSV file
javad_spider.csv_maker()
# Start collecting all the plugins information
plugins = javad_spider.plugin_finder()
javad_spider.fetch_data(plugins)
finally:
google_chrome.quit()
def test_library_keyword_with_named_args(browser: Browser.Browser):
keyword = browser._parse_run_on_failure_keyword(
"Take Screenshot fullPage=True")
assert keyword.name == "take_screenshot"
assert keyword.original_name == "Take Screenshot"
assert keyword.args == tuple()
assert keyword.kwargs == {"fullPage": True}
keyword = browser._parse_run_on_failure_keyword(
"Take Screenshot filename=image fullPage=True")
assert keyword.name == "take_screenshot"
assert keyword.original_name == "Take Screenshot"
assert keyword.args == tuple()
assert keyword.kwargs == {"filename": "image", "fullPage": True}
keyword = browser._parse_run_on_failure_keyword(
"Take Screenshot image-{index} fullPage=True")
assert keyword.name == "take_screenshot"
assert keyword.original_name == "Take Screenshot"
assert keyword.args == ("image-{index}", )
assert keyword.kwargs == {"fullPage": True}
def uploadVideo(self,
video_dir,
videoText,
startTime=0,
endTime=0,
private=True,
test=False,
scheduled=False,
schdate="",
schtime=""):
video_dir = self.downloadIfYoutubeURL(video_dir)
if not video_dir:
return
if self.bot is None:
self.bot = Browser().getBot()
self.webbot = Bot(self.bot)
self.userRequest["dir"] = video_dir
self.checkFileExtensionValid()
self.userRequest["cap"] = self.IO.getHashTagsFromFile()
# Initiate bot if isn't already.
self.bot.get(self.url)
self.userRequest["vidTxt"] = videoText
# Cookies loaded here.
self.cookies = Cookies(self.bot)
self.bot.refresh()
# User now has logged on and can upload videos
time.sleep(3)
self.inputVideo(startTime, endTime)
self.addCaptions()
utils.randomTimeQuery()
if private:
self.webbot.selectPrivateRadio() # private video selection
else:
self.webbot.selectPublicRadio() # public video selection
utils.randomTimeQuery()
if not test:
self.webbot.uploadButtonClick() # upload button
input("Press any button to exit")
def __init__(self, *args, **kwargs):
Browser.__init__(self, *args, **kwargs)
#self.db = kwargs[ "database" ] # Database file
import CONF
#TODO: check if we still need those vars
self.baseURL = CONF.getBaseURLDbCrossway(
) # http://passion-xbmc.org/dbcrossway/
self.catcontentURL = "all/downloads/idparent=%d"
self.xbmctypecontentURL = "all/downloads/type=file;xbmc_type=%s"
self.newcontentURL = "all/downloads/type=file"
self.baseURLDownloadFile = CONF.getBaseURLDownloadFile()
self.baseURLPreviewPicture = CONF.getBaseURLPreviewPicture()
del CONF
self.currentItemId = 0
# Create History instance in order to store browsing history
self.history = History()
def follow_new(self, event):
url, target = self.whichurl(event)
if url:
app = self.context.app
self['background'] = app.prefs.Get(STYLEGROUP, ATEMPPREF)
url = self.context.get_baseurl(url)
from Browser import Browser
Browser(app.root, app).context.load(url)
self['background'] = app.prefs.Get(STYLEGROUP, AHISTPREF)
else:
self.context.viewer.leave_message()
def __init__( self, *args, **kwargs ):
Browser.__init__( self, *args, **kwargs )
#self.db = kwargs[ "database" ] # Database file
import CONF
#TODO: check if we still need those vars
self.baseURL = CONF.getBaseURLDbCrossway() # http://passion-xbmc.org/dbcrossway/
self.catcontentURL = "all/downloads/idparent=%d"
self.xbmctypecontentURL = "all/downloads/type=file;xbmc_type=%s"
self.newcontentURL = "all/downloads/type=file"
self.baseURLDownloadFile = CONF.getBaseURLDownloadFile()
self.baseURLPreviewPicture = CONF.getBaseURLPreviewPicture()
del CONF
self.currentItemId = 0
# Create History instance in order to store browsing history
self.history = History()
def anchor_click_new(self, event):
here = self.text.index("@%d,%d" % (event.x, event.y))
if self.current_index != here:
return
url = self.find_tag_url()
if url:
url, target = self.split_target(url)
self.master.update_idletasks()
from Browser import Browser
app = self.context.app
b = Browser(app.root, app)
b.context.load(self.context.get_baseurl(url))
self.remove_temp_tag(histify=1)
def crawl(self, **args):
source = args['source']
ext = args['ext']
reply_time = args['reply_time']
br = Browser()
page = br.open(self.baseurl)
new_reply_time = reply_time
while True:
links = PageParser.parse(page, source)
for i, link in enumerate(links):
if reply_time < link.reply_time:
if i is 0:
new_reply_time = link.reply_time
self.db.insert('\t'.join([str(link), source, json.dumps(ext)]))
else:
return new_reply_time
try:
page = br.follow_link(text='后页>')
except:
Logger.info('finished!')
break
return new_reply_time
def crawl(self, **args):
source = args['source']
ext = args['ext']
reply_time = args['reply_time']
br = Browser()
page = br.open(self.baseurl)
new_reply_time = reply_time
while True:
links = PageParser.parse(page, source)
for i, link in enumerate(links):
if reply_time < link.reply_time:
if i is 0:
new_reply_time = link.reply_time
self.db.insert('\t'.join(
[str(link), source, json.dumps(ext)]))
else:
return new_reply_time
try:
page = br.follow_link(text='后页>')
except:
Logger.info('finished!')
break
return new_reply_time
def getRequest(self, pluginName, cj=None):
req = Browser(self.bucket, self.getOptions())
if cj:
req.setCookieJar(cj)
else:
req.setCookieJar(CookieJar(pluginName))
return req
def getRequest(self, pluginName, cj=None):
# TODO: mostly obsolete, browser could be removed completely
req = Browser(self.bucket, self.getOptions())
if cj:
req.setCookieJar(cj)
else:
req.setCookieJar(CookieJar(pluginName))
return req
def getRequest(self, pluginName, account=None, type="HTTP", **kwargs):
self.lock.acquire()
options = self.getOptions()
options.update(kwargs) # submit kwargs as additional options
if type == "XDCC":
req = XDCCRequest(self.bucket, options)
else:
req = Browser(self.bucket, options)
if account:
cj = self.getCookieJar(pluginName, account)
req.setCookieJar(cj)
else:
req.setCookieJar(CookieJar(pluginName))
self.lock.release()
return req
def reset( self ):
"""
Reset the browser (back to start page)
"""
Browser.reset()
self.currentItemId = 0
# Import user's grail startup file, defined as
# $GRAILDIR/user/grailrc.py if it exists.
if user_init:
try: import grailrc
except ImportError, e:
# Only catch this is grailrc itself doesn't import,
# otherwise propogate.
if string.split(e.args[0])[-1] != "grailrc":
raise
except:
app.exception_dialog('during import of startup file')
# Load the initial page (command line argument or from preferences)
if not embedded:
from Browser import Browser
browser = Browser(app.root, app, geometry=geometry)
if url:
browser.context.load(url)
elif prefs.GetBoolean('browser', 'load-initial-page'):
browser.home_command()
if not embedded:
# Give the user control
app.go()
class URLReadWrapper:
def __init__(self, api, meta):
self.api = api
self.meta = meta
def crawl(self, **args):
url = args['url']
br = Browser()
return br.open(url, 0.9)
def __init__(self, query, start, length):
self.query = query.replace(' ', '%20')
self.start = start
self.length = length
self.browser = Browser()
def reset( self ):
"""
Reset the browser (back to start page)
"""
Browser.reset()
def setup(self):
self.setWindowTitle("Instrument Browser")
self.ui.authors.addItem("Default instruments")
self.ui.authors.addItem("Own instruments")
self.ui.authors.addItem("All instruments")
Browser.setup(self)
def setup(self):
self.setWindowTitle("Progression Browser")
self.ui.authors.addItem("Default progressions")
self.ui.authors.addItem("Own progressions")
self.ui.authors.addItem("All progressions")
Browser.setup(self)
def scrape(manual_run, input_street_name=None, input_arrondissement=None,
input_street_type=None, input_address=None, input_suite=None, input_direction=None):
'''Function to perform the interaction with the website like a human user'''
AP.reset_dict()
if not input_street_name:
print "Input street name (partial names work best):"
input_street = raw_input("> ")
else:
input_street = input_street_name
print "Input street name (partial names work best):"
print "> " + input_street
input_street = input_street.upper()
# Specifically try for case with "de la" or "des" in street name which
# has tendency to screw things up
input_street_list = [input_street]
if input_street[:6] == "DE LA ":
input_street_list.append(input_street[6:])
elif input_street[:5] == "DE L ":
input_street_list.append(input_street[5:])
elif input_street[:4] == "DES ":
input_street_list.append(input_street[4:])
elif input_street[:3] == "DE ":
input_street_list.append(input_street[3:])
elif input_street[:3] == "DU ":
input_street_list.append(input_street[3:])
selected_nbhood = None
# try best match with full string first and break on success,
# otherwise attempt next best with "de la" or "des" removed
for street_name in input_street_list:
if not selected_nbhood:
print 'Street search: "' + street_name + '"...'
# Emulate web browse instance with Mechanize
BrowserInstance = Browser()
Scraper = BrowserInstance.br
#### Log-in Sequence ####
# Log into Role d'Evaluation Fonciere main page (to set browser
# ASP.NET cookie)
init_url = 'http://evalweb.ville.montreal.qc.ca/default.asp'
Scraper.open(init_url, timeout=TIMEOUT)
# Get list of neighborhoods for entered street name
Scraper.select_form(name='Formulaire')
Scraper.form['text1'] = street_name
street_name = urllib.quote(street_name)
BrowserInstance.mimic_cookie_js(street_name)
response = Scraper.submit().read()
# Look for select_field containing the list of neighboorhoods
# from the returned webpage
soup = BeautifulSoup(response)
neighborhoods_html = soup.find('select', {'id': 'select1'})
try:
nbhood_choices = neighborhoods_html.findAll('option')
nbhood_choices.pop(0) # Remove junk header entry
# Iterate through the displayed options matching them to the
# site URL_ID
nbhd_selection_list = []
for choice in nbhood_choices:
# url id embedded in attributes in a tuple pair
nbhood_url_id = choice.attrs[0][1]
# extract neighborhood name
address_list = AP.parse_nbhoods(choice.text)
address_list.append(nbhood_url_id)
nbhd_selection_list.append(address_list)
if manual_run: # display input selection prompt if run manually
selected_nbhood = selection_prompt(nbhd_selection_list)
selected_nbhood = selected_nbhood - 1
else:
print "Looking for arrondissement: " + input_arrondissement
selected_nbhood = nbhd_search(
nbhd_selection_list, input_arrondissement, input_street,
input_street_type, input_direction)
except Exception, e:
print "Warning: No match found on assessment roll."
time.sleep(1)
