def perform_submission(app_url, testing_secret):
token = None
errors = []
browser = Browser()
browser.open(app_url)
try:
submit_form = browser.getForm(id='briefkasten-form')
except LookupError:
errors.append(
WatchdogError(subject=u"Couldn't find submission form",
message=u"""The contact form was not accessible"""))
return token
submit_form.getControl(
name='message'
).value = u'This is an automated test submission from the watchdog instance.'
submit_form.getControl(name='testing_secret').value = testing_secret
# TODO: submit attachment
submit_form.submit()
response = PyQuery(browser.contents)
token_element = response("#feedback-token")
if token_element is not None:
token = token_element.text()
else:
errors.append(
WatchdogError(
subject="Couldn't get feedback token",
message=
u"The form submission was successful, but no feedback-token was given at %s"
% browser.url))
return token, errors
def perform_submission(app_url, testing_secret):
token = None
errors = []
browser = Browser()
browser.mech_browser.set_handle_robots(False)
try:
browser.open(app_url)
except Exception as exc:
errors.append(WatchdogError(
subject="Couldn't open submission page",
message=u"The attempt to access the submission form resulted in an exception (%s)" % exc))
return token, errors
try:
submit_form = browser.getForm(id='briefkasten-form')
except LookupError:
errors.append(WatchdogError(subject=u"Couldn't find submission form",
message=u"""The contact form was not accessible"""))
return token
submit_form.getControl(name='message').value = u'This is an automated test submission from the watchdog instance.'
submit_form.getControl(name='testing_secret').value = testing_secret
# TODO: submit attachment
submit_form.submit()
response = PyQuery(browser.contents)
token_element = response("#feedback-token")
token = None
if token_element is not None:
token = token_element.text()
if not bool(token):
errors.append(WatchdogError(subject="Couldn't get feedback token",
message=u"The form submission was successful, but no feedback-token was given at %s" % browser.url))
return token, errors
def __init__(self, user_agent=None, wait_time=2, ignore_robots=False):
self.wait_time = wait_time
mech_browser = self._get_mech_browser(user_agent, ignore_robots)
self._browser = Browser(mech_browser=mech_browser)
self._cookie_manager = CookieManager(self._browser.cookies)
self._last_urls = []
def __init__(self, wait_time=2):
self.wait_time = wait_time
self._browser = Browser()
self._cookie_manager = CookieManager(self._browser)
self._last_urls = []
self.links = FindLinks(self)
def __init__(self, url=None, headers=None):
baseBrowser.__init__(self, url=None)
if headers is None: headers = []
self.mech_browser.set_handle_robots(False)
for h in headers:
k, val = h
self.addHeader(k, val)
if url is not None:
self.open(url)
def setupBrowser(auth=None):
"""
Taken from canonical.launchpad.testing.pages:
Create a testbrowser object for use in pagetests.
:param auth: HTTP authentication string. None for the anonymous user, or a
string of the form 'Basic email:password' for an authenticated user.
:return: A `Browser` object.
"""
browser = Browser()
# Set up our Browser objects with handleErrors set to False, since
# that gives a tracebacks instead of unhelpful error messages.
browser.handleErrors = False
if auth is not None:
browser.addHeader("Authorization", auth)
return browser
def __init__(self, user_agent=None, wait_time=2, ignore_robots=False):
self.wait_time = wait_time
mech_browser = self._get_mech_browser(user_agent, ignore_robots)
self._browser = Browser(mech_browser=mech_browser)
self._cookie_manager = CookieManager(self._browser.cookies)
self._last_urls = []
def setUp(self):
super(TestDisplayValue, self).setUp()
app = TestApp()
app.set_next_response(b'''\
<html>
<body>
<form>
<select name="sel1">
<option value="op">Turn</option>
<option value="alt">Alternative</option>
</select>
</form>
</body>
</html>''')
browser = Browser(wsgi_app=app)
browser.open('https://localhost')
self.control = browser.getControl(name='sel1')
def _getPadsPage(self):
username, password, url = self.getUserData()
# login
browser = Browser()
browser.open('%s/ep/account/sign-in' % self.trail_url(url))
browser.getControl(name='email').value = username
browser.getControl(name='password').value = password
browser.getForm(id='signin-form').submit()
# open pads table page
browser.getLink('Pads').click()
return safe_unicode(browser.contents)
def setUp(self):
super(TestMechRepr, self).setUp()
app = TestApp()
app.set_next_response(u'''\
<html>
<body>
<form>
<input name="inp1" type="text" value="Täkst" />
<select name="sel1">
<option value="op">Türn</option>
</select>
<input name="check1" type="checkbox" value="šêlėçtèd" />
<input name="mail1" type="email" value="*****@*****.**" />
<input name="sub1" type="submit" value="Yës" />
</form>
</body>
</html>'''.encode('utf-8'))
self.browser = Browser(wsgi_app=app)
self.browser.open('https://localhost')
def getLink(self, url=None):
if not url: return
# /////////////////////////////////////////////////////////////////////////////
# Scenario 1: do a simple search for all web server logs
# /////////////////////////////////////////////////////////////////////////////
# start search
browser = Browser()
# Get URL
#
# Option A: return all of the matched events
browser.open(url)
ctrl = browser.getControl(name='')
ctrl.value = ''
ctrl = browser.getControl(name='')
ctrl.value = ''
ctrl = browser.getControl(name='submitButtonName')
ctrl.click()
print browser.contents
class TestMechRepr(unittest.TestCase):
"""Testing ..browser.*.mechRepr()."""
def setUp(self):
super(TestMechRepr, self).setUp()
app = TestApp()
app.set_next_response(u'''\
<html>
<body>
<form>
<input name="inp1" type="text" value="Täkst" />
<select name="sel1">
<option value="op">Türn</option>
</select>
<input name="check1" type="checkbox" value="šêlėçtèd" />
<input name="mail1" type="email" value="*****@*****.**" />
<input name="sub1" type="submit" value="Yës" />
</form>
</body>
</html>'''.encode('utf-8'))
self.browser = Browser(wsgi_app=app)
self.browser.open('https://localhost')
def test_TextControl_has_str_mechRepr(self):
mech_repr = self.browser.getControl(name='inp1').mechRepr()
self.assertIsInstance(mech_repr, str)
self.assertEqual(mech_repr, '<TextControl(inp1=Täkst)>')
def test_ItemControl_has_str_mechRepr(self):
option = self.browser.getControl(name='sel1').getControl(value="op")
mech_repr = option.mechRepr()
self.assertIsInstance(mech_repr, str)
self.assertEqual(
mech_repr,
"<Item name='op' id=None contents='Türn' value='op'"
" label='Türn'>")
def test_CheckboxListControl_has_str_mechRepr(self):
from ..browser import CheckboxListControl
ctrl = self.browser.getControl(name='check1')
self.assertIsInstance(ctrl, CheckboxListControl)
mech_repr = ctrl.mechRepr()
self.assertIsInstance(mech_repr, str)
self.assertEqual(mech_repr, '<SelectControl(check1=[*, ambiguous])>')
def test_Control_for_type_email_has_mechRepr(self):
option = self.browser.getControl(name='mail1')
mech_repr = option.mechRepr()
self.assertIsInstance(mech_repr, str)
self.assertEqual(mech_repr, "<EMailControl([email protected])>")
def test_SubmitControl_has_str_mechRepr(self):
mech_repr = self.browser.getControl(name='sub1').mechRepr()
self.assertIsInstance(mech_repr, str)
self.assertEqual(mech_repr, '<SubmitControl(sub1=Yës)>')
def login(self):
br = Browser()
br.mech_browser.set_handle_robots(False)
br.open(self.starturl)
br.getControl('Username:'******'Password:'******'Login').click()
return br
def perform_submission(app_url, testing_secret):
token = None
errors = []
browser = Browser()
browser.open(app_url)
try:
submit_form = browser.getForm(id='briefkasten-form')
except LookupError:
errors.append(WatchdogError(subject=u"Couldn't find submission form",
message=u""""""))
return token
submit_form.getControl(name='message').value = u'This is an automated test submission from the watchdog instance.'
submit_form.getControl(name='testing_secret').value = testing_secret
# TODO: submit attachment
submit_form.submit()
response = PyQuery(browser.contents)
token_element = response("#feedback-token")
if token_element is not None:
token = token_element.text()
else:
errors.append(WatchdogError(subject="Couldn't get feedback token",
message=u"The form submission was successful, but no feedback-token was given at %s" % browser.url))
return token, errors
def __init__(self, username, password, path=None):
"""
@param username: DeviceAtlas username
@param password: DeviceAtlas password
@param path: Path to install API and database files or None to use ~/da_distribution. Path is added to PYTHONPATH.
"""
try:
from zope.testbrowser.browser import Browser
except ImportError:
raise
raise RuntimeError("zope.testbrowser must be installed to automatize DeviceAtlas downloads")
self.browser = Browser()
if not path:
path = os.path.abspath(os.path.join(os.getenv("HOME"), "da_distribution"))
self.path = path
# Wrap things away from Python namespace
self.api_path = os.path.join(path, "da_api")
if not os.path.exists(self.path):
os.makedirs(self.path)
os.makedirs(self.api_path)
if not path in sys.path:
# Include autodownload folder in Python import
sys.path.append(self.path)
self.fname = "da.zip"
self.json_file = "json.zip"
self.raw_json = os.path.join(self.path, "data.json") # Unzipped json database
self.username = username
self.password = password
self.login_url = "http://deviceatlas.com"
self.api_url = url="http://deviceatlas.com/getAPI/python"
def __init__(self, downloads_urls, urls_file, username, password,
dest_dir=None, generate_thumbs=True, font_path=None,
stop_at_quota=500):
self.downloads_urls = downloads_urls
self.urls_file = urls_file
self.username = username
self.password = password
if dest_dir is None:
dest_dir = os.getcwd()
self.dest_dir = dest_dir
self.generate_thumbs = generate_thumbs
self.font_path = font_path
self.stop_at_quota = stop_at_quota
log.info("Starting Browser")
self.browser = Browser("http://oron.com")
self.to_download = 0
self.downloaded = 0
self.download_quota = 0
self.downloads_until_quota_reload = 10
def __init__(self, options):
self.oron_folder_url = options.url
self.title_base = options.title_base
self.links_per_post = options.links_per_post
self.screenshots_dir = options.screenshots
self.output_dir = options.output
# Email Options
self.smtp_host = options.smtp_host
self.smtp_port = options.smtp_port
self.smtp_user = options.smtp_user
self.smtp_pass = options.smtp_pass
self.smtp_from = options.smtp_from
self.smtp_use_tls = options.smtp_use_tls
self.email_recipient = options.email
self.email_initial_text = options.email_initial_text
self.email_ending_text = options.email_ending_text
self.start_on_post = options.start_on_post
log.info("Starting Browser")
self.browser = Browser("http://oron.com")
self.total_links = 0
self.links = {}
class ZopeTestBrowser(ElementPresentMixIn, DriverAPI):
driver_name = "zope.testbrowser"
def __init__(self, user_agent=None, wait_time=2, ignore_robots=False):
self.wait_time = wait_time
mech_browser = self._get_mech_browser(user_agent, ignore_robots)
self._browser = Browser(mech_browser=mech_browser)
self._cookie_manager = CookieManager(self._browser.cookies)
self._last_urls = []
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
pass
def visit(self, url):
self._browser.open(url)
def back(self):
self._last_urls.insert(0, self.url)
self._browser.goBack()
def forward(self):
try:
self.visit(self._last_urls.pop())
except IndexError:
pass
def reload(self):
self._browser.reload()
def quit(self):
pass
@property
def htmltree(self):
return lxml.html.fromstring(self.html.decode('utf-8'))
@property
def title(self):
return self._browser.title
@property
def html(self):
return self._browser.contents
@property
def url(self):
return self._browser.url
def find_option_by_value(self, value):
html = self.htmltree
element = html.xpath('//option[@value="%s"]' % value)[0]
control = self._browser.getControl(element.text)
return ElementList([ZopeTestBrowserOptionElement(control, self)],
find_by="value",
query=value)
def find_option_by_text(self, text):
html = self.htmltree
element = html.xpath('//option[normalize-space(text())="%s"]' %
text)[0]
control = self._browser.getControl(element.text)
return ElementList([ZopeTestBrowserOptionElement(control, self)],
find_by="text",
query=text)
def find_by_css(self, selector):
xpath = CSSSelector(selector).path
return self.find_by_xpath(xpath,
original_find="css",
original_selector=selector)
def find_by_xpath(self, xpath, original_find=None, original_selector=None):
html = self.htmltree
elements = []
for xpath_element in html.xpath(xpath):
if self._element_is_link(xpath_element):
return self._find_links_by_xpath(xpath)
elif self._element_is_control(xpath_element):
return self.find_by_name(xpath_element.name)
else:
elements.append(xpath_element)
find_by = original_find or "xpath"
query = original_selector or xpath
return ElementList(
[ZopeTestBrowserElement(element, self) for element in elements],
find_by=find_by,
query=query)
def find_by_tag(self, tag):
return self.find_by_xpath('//%s' % tag,
original_find="tag",
original_selector=tag)
def find_by_value(self, value):
return self.find_by_xpath('//*[@value="%s"]' % value,
original_find="value",
original_selector=value)
def find_by_text(self, text):
return self.find_by_xpath('//*[text()="%s"]' % text,
original_find="text",
original_selector=text)
def find_by_id(self, id_value):
return self.find_by_xpath('//*[@id="%s"][1]' % id_value,
original_find="id",
original_selector=id_value)
def find_by_name(self, name):
elements = []
index = 0
while True:
try:
control = self._browser.getControl(name=name, index=index)
elements.append(control)
index += 1
except LookupError:
break
return ElementList([
ZopeTestBrowserControlElement(element, self)
for element in elements
],
find_by="name",
query=name)
def find_link_by_text(self, text):
return self._find_links_by_xpath("//a[text()='%s']" % text)
def find_link_by_href(self, href):
return self._find_links_by_xpath("//a[@href='%s']" % href)
def find_link_by_partial_href(self, partial_href):
return self._find_links_by_xpath("//a[contains(@href, '%s')]" %
partial_href)
def find_link_by_partial_text(self, partial_text):
return self._find_links_by_xpath(
"//a[contains(normalize-space(.), '%s')]" % partial_text)
def fill(self, name, value):
self.find_by_name(name=name).first._control.value = value
def fill_form(self, field_values):
for name, value in field_values.items():
element = self.find_by_name(name)
control = element.first._control
if control.type == 'checkbox':
if value:
control.value = control.options
else:
control.value = []
elif control.type == 'radio':
control.value = [
option for option in control.options if option == value
]
elif control.type == 'select':
element.select(value)
else:
# text, textarea, password, tel
control.value = value
def choose(self, name, value):
control = self._browser.getControl(name=name)
control.value = [
option for option in control.options if option == value
]
def check(self, name):
control = self._browser.getControl(name=name)
control.value = control.options
def uncheck(self, name):
control = self._browser.getControl(name=name)
control.value = []
def attach_file(self, name, file_path):
filename = file_path.split('/')[-1]
control = self._browser.getControl(name=name)
content_type, _ = mimetypes.guess_type(file_path)
control.add_file(open(file_path), content_type, filename)
def _find_links_by_xpath(self, xpath):
html = self.htmltree
links = html.xpath(xpath)
return ElementList(
[ZopeTestBrowserLinkElement(link, self) for link in links],
find_by="xpath",
query=xpath)
def select(self, name, value):
self.find_by_name(name).first._control.value = [value]
def is_text_present(self, text, wait_time=None):
wait_time = wait_time or self.wait_time
end_time = time.time() + wait_time
while time.time() < end_time:
if self._is_text_present(text):
return True
return False
def _is_text_present(self, text):
try:
body = self.find_by_tag('body').first
return text in body.text
except ElementDoesNotExist:
# This exception will be thrown if the body tag isn't present
# This has occasionally been observed. Assume that the
# page isn't fully loaded yet
return False
def is_text_not_present(self, text, wait_time=None):
wait_time = wait_time or self.wait_time
end_time = time.time() + wait_time
while time.time() < end_time:
if not self._is_text_present(text):
return True
return False
def _element_is_link(self, element):
return element.tag == 'a'
def _element_is_control(self, element):
return hasattr(element, 'type')
def _get_mech_browser(self, user_agent, ignore_robots):
mech_browser = mechanize.Browser()
if user_agent is not None:
mech_browser.addheaders = [
("User-agent", user_agent),
]
if ignore_robots:
mech_browser.set_handle_robots(False)
return mech_browser
@property
def cookies(self):
return self._cookie_manager
def __init__(self):
self._browser = Browser()
self._cookie_manager = CookieManager(self._browser.cookies)
self._last_urls = []
class OronDownloader(object):
def __init__(self, downloads_urls, urls_file, username, password,
dest_dir=None, generate_thumbs=True, font_path=None,
stop_at_quota=500):
self.downloads_urls = downloads_urls
self.urls_file = urls_file
self.username = username
self.password = password
if dest_dir is None:
dest_dir = os.getcwd()
self.dest_dir = dest_dir
self.generate_thumbs = generate_thumbs
self.font_path = font_path
self.stop_at_quota = stop_at_quota
log.info("Starting Browser")
self.browser = Browser("http://oron.com")
self.to_download = 0
self.downloaded = 0
self.download_quota = 0
self.downloads_until_quota_reload = 10
def login(self):
if self.browser and self.browser.contents:
if self.username in self.browser.contents and \
'Logout' in self.browser.contents:
log.info("Already logged in!")
return
log.info("Logging into Oron.com")
self.browser.open('http://oron.com/login')
self.browser.getControl(name='login').value = self.username
self.browser.getControl(name='password').value = self.password
self.browser.getForm(name='FL').submit(' Submit ')
if self.username not in self.browser.contents:
log.error("Failed to login...")
print self.browser.contents
sys.exit(1)
log.info("Logged in successfully")
self.load_download_quota()
def load_download_quota(self):
log.info("Loading download quota...")
self.browser.open("http://oron.com/?op=my_account")
doc = etree.HTML(self.browser.contents)
quota_string_match = doc.xpath('//form/table/tr[3]/td[2]/text()')
if quota_string_match:
self.download_quota = int(quota_string_match[0].split()[0])
log.info("You have %d Mb of download quota.", self.download_quota)
else:
log.error("Failed to retrieve download quota!")
sys.exit(1)
def check_quota(self):
if self.download_quota <= self.stop_at_quota:
log.info("Download quota is now %d. Stop downloading for now...",
self.download_quota)
sys.exit(0)
if not self.downloads_until_quota_reload:
self.downloads_until_quota_reload = 10
self.load_download_quota()
else:
self.downloads_until_quota_reload -= 1
def download(self):
self.login()
self.check_quota()
if self.urls_file:
log.info("Opening download URLs file: %s", self.urls_file)
self.find_download_links_from_file()
else:
log.info("Processing %d oron folder urls", len(self.downloads_urls))
for url in self.downloads_urls:
log.info("Opening downloads URL: %s", url)
self.browser.open(url)
self.find_download_links(self.browser.contents)
def find_download_links_from_file(self):
urls = open(self.urls_file, 'r').readlines(True)
self.to_download = len([u for u in urls if u.strip()])
log.info("Processing %d oron file urls", self.to_download)
for url in urls:
if not url.strip():
continue
url = url.strip()
log.info('-'*78)
log.info("[%d/%d] Opening URL: %s", self.downloaded+1, self.to_download, url)
attempts = 0
while True:
try:
self.browser.open(url.strip())
doc = etree.HTML(self.browser.contents)
break
except socket.timeout:
if attempts > 2:
break
log.info("Socket timeout. retrying...")
attempts += 1
if attempts > 2:
log.info("Too many timeouts. Continue to next...")
continue
info_node = node = doc.xpath('//div[@class="content"]/form/table/tr/td')
#log.info("Info node match: %s", info_node)
if info_node:
info_node = info_node[0]
else:
file_not_found_match = doc.xpath('//h2/text()')
if file_not_found_match:
log.error("File was not found on server. Skipping...")
else:
log.error("Could not find out the filename. Skipping...")
continue
filename_match = info_node.xpath('./b/text()')
#log.info("filename match is %s", filename_match)
if filename_match:
filename = filename_match[0].encode('utf8')
size = info_node.xpath('./text()')[-1].strip().split('File size:')[-1].strip().upper()
fpath = os.path.join(self.dest_dir, filename)
spath = os.path.join(self.dest_dir, 'seen', filename)
upath = os.path.join(self.dest_dir, 'upped', filename)
uspath = os.path.join(self.dest_dir, 'upped-n-seen', filename)
usnpath = os.path.join(self.dest_dir, 'upped-not-seen', filename)
process_next = False
for path in (fpath, spath, upath, uspath, usnpath):
try:
if os.path.isfile(path):
if self.humanize_bytes(os.path.getsize(path)) == size:
log.info("Filename %s already downloaded(%s). Skipping...",
filename, path)
self.downloaded += 1
process_next = True
break
log.info("Downloaded %s/%s of %s so far. Continuing...",
self.humanize_bytes(os.path.getsize(path)), size,
filename)
except UnicodeEncodeError:
print 'Unicode error for filename', filename
process_next = True
break
if process_next:
continue
errors = 0
while errors <= 3:
log.info("Processing[%s] %s from %s", errors, filename, url)
try:
self.download_link(url)
break
except (socket.error, urllib2.URLError), err:
log.error("Failed to download link: %s", err)
errors += 1
else:
log.info("Too many errors, continue to next link")
continue
if self.generate_thumbs:
self.generate_thumb(filename)
self.download_quota -= round(float(
os.path.getsize(os.path.join(self.dest_dir, filename))
)/1024/1024, 1)
if self.download_quota <= self.stop_at_quota:
log.info("Download quota is now %d. Stop downloading for now...",
self.download_quota)
sys.exit(0)
if not self.downloads_until_quota_reload:
self.downloads_until_quota_reload = 10
self.load_download_quota()
else:
self.downloads_until_quota_reload -= 1
class DAInstaller:
""" Automatically download DeviceAtlas API bundle and extract.
DeviceAtlas Python API files are downloaded from
and installed to ${your home folder}/da_distribution.
This is to keep proprietary files away from your own
(potentially open source) source tree.
zc.testbrowser class is used to mimic the real web browser.
To install zope.testbrowser, type:
easy_install zope.testbrowser
You might or might not want to do this, since Zope dependencies are quite heavy.
"""
def __init__(self, username, password, path=None):
"""
@param username: DeviceAtlas username
@param password: DeviceAtlas password
@param path: Path to install API and database files or None to use ~/da_distribution. Path is added to PYTHONPATH.
"""
try:
from zope.testbrowser.browser import Browser
except ImportError:
raise
raise RuntimeError("zope.testbrowser must be installed to automatize DeviceAtlas downloads")
self.browser = Browser()
if not path:
path = os.path.abspath(os.path.join(os.getenv("HOME"), "da_distribution"))
self.path = path
# Wrap things away from Python namespace
self.api_path = os.path.join(path, "da_api")
if not os.path.exists(self.path):
os.makedirs(self.path)
os.makedirs(self.api_path)
if not path in sys.path:
# Include autodownload folder in Python import
sys.path.append(self.path)
self.fname = "da.zip"
self.json_file = "json.zip"
self.raw_json = os.path.join(self.path, "data.json") # Unzipped json database
self.username = username
self.password = password
self.login_url = "http://deviceatlas.com"
self.api_url = url="http://deviceatlas.com/getAPI/python"
def login(self):
"""
"""
self.browser.open(self.login_url)
if "Signed in as" in self.browser.contents:
return # already logged in
field = self.browser.getControl(name="name")
field.value = self.username
field = self.browser.getControl(name="pass")
field.value = self.password
submit = self.browser.getControl("Go")
submit.click()
if not "http://deviceatlas.com/user" in self.browser.url:
raise RuntimeError("Could not log in to deviceatlas.com with username " + self.username)
#print self.browser.url
#print self.browser.headers
#print self.browser.contents
def need_download(self):
"""
@return: True if zip download needed
"""
if not os.path.exists(os.path.join(self.path, self.fname)):
return True
return False
def need_unzip(self):
"""
@return: True if zip extract needed
"""
if not os.path.exists(os.path.join(self.api_path, "api.py")):
return True
return False
def download_zip(self):
import urllib2
self.browser.open("http://deviceatlas.com/getAPI/python")
assert self.browser.headers["Content-Type"] == "application/zip" # Check that we fetch the correct file
assert len(self.browser.contents) > 0, "Could not download da.zip API distribution"
o = open(os.path.join(self.path, self.fname), "wb")
o.write(self.browser.contents)
o.close()
def unzip_file_into_dir(self, file, dir, override_name=None):
import zipfile
if not os.path.exists(dir):
os.mkdir(dir)
zfobj = zipfile.ZipFile(file)
for name in zfobj.namelist():
if name.endswith('/'):
os.mkdir(os.path.join(dir, name))
else:
if override_name:
# Set extracted file name
fname = override_name
else:
fname = name
outfile = open(os.path.join(dir, fname), 'wb')
outfile.write(zfobj.read(name))
outfile.close()
def need_database(self):
if not os.path.exists(self.raw_json):
return True
return False
def download_database(self):
self.browser.open("http://deviceatlas.com")
self.browser.getLink("My account").click()
# Assume Community ZIP link
self.browser.getLink(url="/zip", index=0).click()
# Check that we fetch the correct file
assert self.browser.headers["Content-Type"] == "application/zip", "Assume zip content:" + str(self.browser.headers)
o = open(os.path.join(self.path, self.json_file), "wb")
o.write(self.browser.contents)
o.close()
def proceed(self):
""" Download & unzip DA Python API files if needed.
@return: Path to Downloaded DeviceAtlas JSON data file
"""
if self.need_download():
self.login()
self.download_zip()
if self.need_unzip():
self.unzip_file_into_dir(os.path.join(self.path, self.fname), self.api_path)
# create __init__.py to mark the module
f = open(os.path.join(self.api_path, "__init__.py"), "wt")
f.write("# This package is automatically generated by DAAPIInstaller")
f.close()
if self.need_database():
self.login()
self.download_database()
self.unzip_file_into_dir(os.path.join(self.path, self.json_file), self.path, "data.json")
return self.raw_json
class Administrator():
'''An automated Google Apps administrator.'''
def __init__(self, domain, username, password):
'''Create an administrator, then log in with the given credentials.'''
self.domain = domain
self.browser = Browser()
self.login(username, password)
def login(self, username, password):
'''Log in with the given username and password.'''
self.browser.open('https://www.google.com/a/%s/ServiceLogin'
% self.domain)
form = self.browser.getForm(id='gaia_loginform')
# domain is automatically added
form.getControl(name='Email').value = username
form.getControl(name='Passwd').value = password
form.submit()
def go_to_group(self, group):
'''Open the groups's page.'''
self.browser.open('https://www.google.com/a/cpanel/%s/Group?groupId=%s'
% (self.domain, group))
def users_in_group(self, group):
'''Return the set of users in the group.'''
self.go_to_group(group)
# Add all users in the current (first page).
users = emails_in_html(self.browser.contents)
try:
# Go to next pages and add all of them.
while True:
self.browser.getLink(text='Next').click()
users.update(emails_in_html(self.browser.contents))
except LinkNotFoundError:
pass
return users
def add_users_to_group(self, users, group):
'''Add the users to the group.'''
unique_users = set(users)
current_users = self.users_in_group(group)
new_users = unique_users - current_users
self.go_to_group(group)
for chunk in split_users(new_users):
form = self.browser.getForm(id='addmember')
form.getControl(name='members').value = ','.join(chunk)
# Click this button to submit the form
form.getControl(name='add').click()
def remove_users_from_group(self, users, group):
'''Remove the users from the group.'''
unique_users = set(users)
self.go_to_group(group)
tree = html.fromstring(self.browser.contents)
form = self.browser.getForm(id='list')
for hidden_input in tree.cssselect('form#list td input[type="hidden"]'):
print 'checking', hidden_input.value
if hidden_input.value in unique_users:
print 'removing', hidden_input.value
checkbox = form.getControl(name=hidden_input.getnext().name)
print checkbox.mech_control
print 'submitting'
form.getControl(name='remove', index=0).click()
class ZopeTestBrowser(DriverAPI):
def __init__(self):
self._browser = Browser()
self._last_urls = []
def visit(self, url):
self._browser.open(url)
def back(self):
self._last_urls.insert(0, self.url)
self._browser.goBack()
def forward(self):
try:
self.visit(self._last_urls.pop())
except IndexError:
pass
def reload(self):
self._browser.reload()
def quit(self):
pass
@property
def title(self):
return self._browser.title
@property
def html(self):
return self._browser.contents
@property
def url(self):
return self._browser.url
def find_option_by_value(self, value):
html = lxml.html.fromstring(self.html)
element = html.xpath('//option[@value="%s"]' % value)[0]
control = self._browser.getControl(element.text)
return ElementList([ZopeTestBrowserOptionElement(control, self)])
def find_option_by_text(self, text):
html = lxml.html.fromstring(self.html)
element = html.xpath('//option[normalize-space(text())="%s"]' % text)[0]
control = self._browser.getControl(element.text)
return ElementList([ZopeTestBrowserOptionElement(control, self)])
def find_by_css(self, selector):
xpath = CSSSelector(selector).path
return self.find_by_xpath(xpath)
find_by_css_selector = warn_deprecated(find_by_css, 'find_by_css_selector')
def find_by_xpath(self, xpath):
html = lxml.html.fromstring(self.html)
elements = []
for xpath_element in html.xpath(xpath):
if self._element_is_link(xpath_element):
return self.find_link_by_text(xpath_element.text)
elif self._element_is_control(xpath_element):
return self.find_by_name(xpath_element.name)
else:
elements.append(xpath_element)
return ElementList([ZopeTestBrowserElement(element, self) for element in elements])
def find_by_tag(self, tag):
return self.find_by_xpath('//%s' % tag)
def find_by_id(self, id_value):
return self.find_by_xpath('//*[@id="%s"][1]' % id_value)
def find_by_name(self, name):
elements = []
index = 0
while True:
try:
control = self._browser.getControl(name=name, index=index)
elements.append(control)
index += 1
except IndexError:
break
return ElementList([ZopeTestBrowserControlElement(element, self) for element in elements])
def find_link_by_text(self, text):
return self._find_links_by_xpath("//a[text()='%s']" % text)
def find_link_by_href(self, href):
return self._find_links_by_xpath("//a[@href='%s']" % href)
def fill(self, name, value):
self.find_by_name(name=name).first._control.value = value
fill_in = warn_deprecated(fill, 'fill_in')
def choose(self, name):
control = self._browser.getControl(name=name)
control.value = control.options
check = choose
def uncheck(self, name):
control = self._browser.getControl(name=name)
control.value = []
def attach_file(self, name, file_path):
control = self._browser.getControl(name=name)
content_type, _ = mimetypes.guess_type(file_path)
control.add_file(open(file_path), content_type, None)
def _find_links_by_xpath(self, xpath):
html = lxml.html.fromstring(self.html)
links = html.xpath(xpath)
return ElementList([ZopeTestBrowserLinkElement(link, self) for link in links])
def select(self, name, value):
self.find_by_name(name).first._control.value = [value,]
def _element_is_link(self, element):
return element.tag == 'a'
def _element_is_control(self, element):
return hasattr(element, 'type')
def getBrowser(self,
username=TEST_USER_NAME,
password=TEST_USER_PASSWORD,
loggedIn=True):
# Instantiate and return a testbrowser for convenience
browser = Browser(self.portal)
browser.addHeader("Accept-Language", "en-US")
browser.handleErrors = False
if loggedIn:
browser.open(self.portal.absolute_url())
browser.getControl("Login Name").value = username
browser.getControl("Password").value = password
browser.getControl("Log in").click()
self.assertTrue("You are now logged in" in browser.contents)
return browser
def fetch_data(self):
url, username, password = self._url, self._username, self._password
login_url = '/tutos/php/bookinginserter/login.php'
showteamhours_url = '/tutos/php/bookinginserter/showteamhours.php'
browser = Browser(url)
browser.getControl(name='username').value = username
browser.getControl(name='password').value = password
browser.getForm(action=login_url).submit()
browser.getLink(url='showteamhours.php').click()
start_date_value = browser.getControl(name='startdate').options[0]
browser.getControl(name='startdate').value = [start_date_value]
end_date_value = browser.getControl(name='enddate').options[-1]
browser.getControl(name='enddate').value = [end_date_value]
browser.getForm(action=showteamhours_url).submit()
self._from_date = start_date_value
self._to_date = end_date_value
self._data = browser.contents
def __init__(self, user_agent=None):
mech_browser = self._get_mech_browser(user_agent)
self._browser = Browser(mech_browser=mech_browser)
self._cookie_manager = CookieManager(self._browser.cookies)
self._last_urls = []
def __init__(self):
self._browser = Browser()
self._last_urls = []
def __init__(self, *args, **kwargs):
kwargs['mech_browser'] = InterceptBrowser()
ZopeTestbrowser.__init__(self, *args, **kwargs)
#imports para a geracao dos documentos
import urllib
import cStringIO
from appy.pod.renderer import Renderer
#imports para o maps
import sys
import six
import base64
import simplejson as json
from lacunarestpki import *
from zope.testbrowser.browser import Browser
browser = Browser()
restpki_access_token = ''
restpki_url = 'https://pki.rest/'
restpki_client = RestPkiClient(restpki_url, restpki_access_token)
class ISAPLTool(Interface):
""" Marker interface for SAPL Tool.
"""
pass
class SAPLTool(UniqueObject, SimpleItem, ActionProviderBase):
__implements__ = (ISAPLTool)
def create_posts(self):
self.grab_links()
filenames = sorted(self.links.keys())
chunker = Chunker(self.links_per_post)
n = 1
if n < self.start_on_post:
log.info("Skipping posts %d to %d", n, self.start_on_post)
for chunk in chunker(filenames):
if n < self.start_on_post:
n += 1
continue
log.info("Processing post %s of %s", n, self.total_posts)
title = self.title_base + ' - %d of %d' % (n, self.total_posts)
links = []
for filename in chunk:
screenshot_name = filename + '.jpg'
screenshot_path = os.path.join(self.screenshots_dir, screenshot_name)
image_html = "<em>Missing Image preview for <b>%s</b></em>" % filename
if os.path.isfile(screenshot_path):
log.info("Uploading screenshot %s", screenshot_name)
browser = Browser("http://www.freeporndumpster.com/legacy.php")
caption_control = browser.getControl(name='imagename[]', index=0)
caption_control.value = self.links[filename]['href']
file_control = browser.getControl(name='images[]', index=0)
file_control.add_file(
open(screenshot_path),
"image/jpeg", screenshot_name
)
browser.getControl('upload').click()
doc = etree.HTML(browser.contents)
image_html = "<em>Missing Image html for <b>%s</b></em>" % filename
image_html_match = doc.xpath('//table/tr/td/p[contains(., "website")]/textarea/text()')
if image_html_match:
image_html = image_html_match[0]
else:
log.error("Failed to get uploaded image html")
else:
log.warn("There's no screenshot for filename %s", filename)
links.append((
filename,
self.links[filename]['size'],
self.links[filename]['href'],
image_html
))
html = HTML_TEMPLATE.render(
links=links, initial_text=self.email_initial_text,
ending_text=self.email_ending_text
)
open(os.path.join(self.output_dir, title+'.txt'), 'w').write(html)
n += 1
server = None
email = MIMEMultipart('alternative')
email['Subject'] = title
email['From'] = '*****@*****.**'
email['To'] = self.email_recipient
email.attach(MIMEText(html, 'html'))
log.info("Sending post email")
try:
try:
# Python 2.6
server = smtplib.SMTP(self.smtp_host, self.smtp_port, timeout=60)
except:
# Python 2.5
server = smtplib.SMTP(self.smtp_host, self.smtp_port)
except Exception, err:
log.error("There was an error sending the notification email: %s", err)
if server is None:
log.error("Failed to setup SMTP server")
continue
security_enabled = self.smtp_use_tls
if security_enabled:
server.ehlo()
if not server.esmtp_features.has_key('starttls'):
log.warning("TLS/SSL enabled but server does not support it")
else:
server.starttls()
server.ehlo()
if self.smtp_user and self.smtp_pass:
try:
server.login(self.smtp_user, self.smtp_pass)
except smtplib.SMTPHeloError, err:
log.error("The server didn't reply properly to the helo "
"greeting: %s", err)
except smtplib.SMTPAuthenticationError, err:
log.error("The server didn't accept the username/password "
"combination: %s", err)
class Blogger(object):
def __init__(self, options):
self.oron_folder_url = options.url
self.title_base = options.title_base
self.links_per_post = options.links_per_post
self.screenshots_dir = options.screenshots
self.output_dir = options.output
# Email Options
self.smtp_host = options.smtp_host
self.smtp_port = options.smtp_port
self.smtp_user = options.smtp_user
self.smtp_pass = options.smtp_pass
self.smtp_from = options.smtp_from
self.smtp_use_tls = options.smtp_use_tls
self.email_recipient = options.email
self.email_initial_text = options.email_initial_text
self.email_ending_text = options.email_ending_text
self.start_on_post = options.start_on_post
log.info("Starting Browser")
self.browser = Browser("http://oron.com")
self.total_links = 0
self.links = {}
def login(self):
if self.browser and self.browser.contents:
if self.username in self.browser.contents and \
'Logout' in self.browser.contents:
log.info("Already logged in!")
return
log.info("Logging into Oron.com")
self.browser.open('http://oron.com/login')
self.browser.getControl(name='login').value = self.username
self.browser.getControl(name='password').value = self.password
self.browser.getForm(name='FL').submit(' Submit ')
if self.username not in self.browser.contents:
log.error("Failed to login...")
print self.browser.contents
sys.exit(1)
log.info("Logged in successfully")
def grab_links(self):
log.info("Loading oron folder url: %s", self.oron_folder_url)
self.browser.open(self.oron_folder_url)
doc = etree.HTML(self.browser.contents)
self.total_links = len(doc.xpath(
'//table[@class="tbl2"]/tr/td[1]/a[@target="_blank"]/small'
))
self.total_posts = int(round(self.total_links/(self.links_per_post*1.0)))
log.info("Found %d links to create %s blog posts.",
self.total_links, self.total_posts)
trs = doc.xpath('//table[@class="tbl2"]/tr')
for tr in trs:
link_td = tr.xpath('td[1]/a')
size_td = tr.xpath('td[2]/small/nobr/text()')
if not link_td or not size_td:
continue
filename_search = link_td[0].xpath('small/text()')
if not filename_search:
continue
filename = filename_search[0]
href = link_td[0].attrib['href']
size = size_td[0].upper()
self.links[filename] = {
'href': href, 'size': size, 'filename': filename
}
def create_posts(self):
self.grab_links()
filenames = sorted(self.links.keys())
chunker = Chunker(self.links_per_post)
n = 1
if n < self.start_on_post:
log.info("Skipping posts %d to %d", n, self.start_on_post)
for chunk in chunker(filenames):
if n < self.start_on_post:
n += 1
continue
log.info("Processing post %s of %s", n, self.total_posts)
title = self.title_base + ' - %d of %d' % (n, self.total_posts)
links = []
for filename in chunk:
screenshot_name = filename + '.jpg'
screenshot_path = os.path.join(self.screenshots_dir, screenshot_name)
image_html = "<em>Missing Image preview for <b>%s</b></em>" % filename
if os.path.isfile(screenshot_path):
log.info("Uploading screenshot %s", screenshot_name)
browser = Browser("http://www.freeporndumpster.com/legacy.php")
caption_control = browser.getControl(name='imagename[]', index=0)
caption_control.value = self.links[filename]['href']
file_control = browser.getControl(name='images[]', index=0)
file_control.add_file(
open(screenshot_path),
"image/jpeg", screenshot_name
)
browser.getControl('upload').click()
doc = etree.HTML(browser.contents)
image_html = "<em>Missing Image html for <b>%s</b></em>" % filename
image_html_match = doc.xpath('//table/tr/td/p[contains(., "website")]/textarea/text()')
if image_html_match:
image_html = image_html_match[0]
else:
log.error("Failed to get uploaded image html")
else:
log.warn("There's no screenshot for filename %s", filename)
links.append((
filename,
self.links[filename]['size'],
self.links[filename]['href'],
image_html
))
html = HTML_TEMPLATE.render(
links=links, initial_text=self.email_initial_text,
ending_text=self.email_ending_text
)
open(os.path.join(self.output_dir, title+'.txt'), 'w').write(html)
n += 1
server = None
email = MIMEMultipart('alternative')
email['Subject'] = title
email['From'] = '*****@*****.**'
email['To'] = self.email_recipient
email.attach(MIMEText(html, 'html'))
log.info("Sending post email")
try:
try:
# Python 2.6
server = smtplib.SMTP(self.smtp_host, self.smtp_port, timeout=60)
except:
# Python 2.5
server = smtplib.SMTP(self.smtp_host, self.smtp_port)
except Exception, err:
log.error("There was an error sending the notification email: %s", err)
if server is None:
log.error("Failed to setup SMTP server")
continue
security_enabled = self.smtp_use_tls
if security_enabled:
server.ehlo()
if not server.esmtp_features.has_key('starttls'):
log.warning("TLS/SSL enabled but server does not support it")
else:
server.starttls()
server.ehlo()
if self.smtp_user and self.smtp_pass:
try:
server.login(self.smtp_user, self.smtp_pass)
except smtplib.SMTPHeloError, err:
log.error("The server didn't reply properly to the helo "
"greeting: %s", err)
except smtplib.SMTPAuthenticationError, err:
log.error("The server didn't accept the username/password "
"combination: %s", err)
class ZopeTestBrowser(DriverAPI):
def __init__(self):
self._browser = Browser()
self._cookie_manager = CookieManager(self._browser.cookies)
self._last_urls = []
def visit(self, url):
self._browser.open(url)
def back(self):
self._last_urls.insert(0, self.url)
self._browser.goBack()
def forward(self):
try:
self.visit(self._last_urls.pop())
except IndexError:
pass
def reload(self):
self._browser.reload()
def quit(self):
pass
@property
def title(self):
return self._browser.title
@property
def html(self):
return self._browser.contents
@property
def url(self):
return self._browser.url
def find_option_by_value(self, value):
html = lxml.html.fromstring(self.html)
element = html.xpath('//option[@value="%s"]' % value)[0]
control = self._browser.getControl(element.text)
return ElementList([ZopeTestBrowserOptionElement(control, self)])
def find_option_by_text(self, text):
html = lxml.html.fromstring(self.html)
element = html.xpath('//option[normalize-space(text())="%s"]' % text)[0]
control = self._browser.getControl(element.text)
return ElementList([ZopeTestBrowserOptionElement(control, self)])
def find_by_css(self, selector):
xpath = CSSSelector(selector).path
return self.find_by_xpath(xpath)
find_by_css_selector = warn_deprecated(find_by_css, 'find_by_css_selector')
def find_by_xpath(self, xpath):
html = lxml.html.fromstring(self.html)
elements = []
for xpath_element in html.xpath(xpath):
if self._element_is_link(xpath_element):
return self.find_link_by_text(xpath_element.text)
elif self._element_is_control(xpath_element):
return self.find_by_name(xpath_element.name)
else:
elements.append(xpath_element)
return ElementList([ZopeTestBrowserElement(element, self) for element in elements])
def find_by_tag(self, tag):
return self.find_by_xpath('//%s' % tag)
def find_by_value(self, value):
return self.find_by_xpath('//*[@value="%s"]' % value)
def find_by_id(self, id_value):
return self.find_by_xpath('//*[@id="%s"][1]' % id_value)
def find_by_name(self, name):
elements = []
index = 0
while True:
try:
control = self._browser.getControl(name=name, index=index)
elements.append(control)
index += 1
except IndexError:
break
return ElementList([ZopeTestBrowserControlElement(element, self) for element in elements])
def find_link_by_text(self, text):
return self._find_links_by_xpath("//a[text()='%s']" % text)
def find_link_by_href(self, href):
return self._find_links_by_xpath("//a[@href='%s']" % href)
def find_link_by_partial_text(self, partial_text):
return self._find_links_by_xpath("//a[contains(text(), '%s')]" % partial_text)
def fill(self, name, value):
self.find_by_name(name=name).first._control.value = value
fill_in = warn_deprecated(fill, 'fill_in')
def choose(self, name, value):
control = self._browser.getControl(name=name)
control.value = [option for option in control.options if option == value]
def check(self, name):
control = self._browser.getControl(name=name)
control.value = control.options
def uncheck(self, name):
control = self._browser.getControl(name=name)
control.value = []
def attach_file(self, name, file_path):
control = self._browser.getControl(name=name)
content_type, _ = mimetypes.guess_type(file_path)
control.add_file(open(file_path), content_type, None)
def _find_links_by_xpath(self, xpath):
html = lxml.html.fromstring(self.html)
links = html.xpath(xpath)
return ElementList([ZopeTestBrowserLinkElement(link, self) for link in links])
def select(self, name, value):
self.find_by_name(name).first._control.value = [value]
def _element_is_link(self, element):
return element.tag == 'a'
def _element_is_control(self, element):
return hasattr(element, 'type')
@property
def cookies(self):
return self._cookie_manager
def __init__(self):
self._browser = Browser()
def __init__(self, domain, username, password):
'''Create an administrator, then log in with the given credentials.'''
self.domain = domain
self.browser = Browser()
self.login(username, password)
class ZopeTestBrowser(DriverAPI):
def __init__(self):
self._browser = Browser()
def visit(self, url):
self._browser.open(url)
def quit(self):
pass
@property
def title(self):
return self._browser.title
@property
def html(self):
return self._browser.contents
@property
def url(self):
return self._browser.url
def find_option_by_value(self, value):
html = lxml.html.fromstring(self.html)
element = html.xpath('//option[@value="%s"]' % value)[0]
control = self._browser.getControl(element.text)
return ElementList([ZopeTestBrowserOptionElement(control)])
def find_option_by_text(self, text):
html = lxml.html.fromstring(self.html)
element = html.xpath('//option[normalize-space(text())="%s"]' % text)[0]
control = self._browser.getControl(element.text)
return ElementList([ZopeTestBrowserOptionElement(control)])
def find_by_css_selector(self, selector):
xpath = CSSSelector(selector).path
return self.find_by_xpath(xpath)
def find_by_xpath(self, xpath):
html = lxml.html.fromstring(self.html)
elements = []
for xpath_element in html.xpath(xpath):
if self._element_is_link(xpath_element):
return self.find_link_by_text(xpath_element.text)
elif self._element_is_control(xpath_element):
return self.find_by_name(xpath_element.name)
else:
elements.append(xpath_element)
return ElementList([ZopeTestBrowserElement(element) for element in elements])
def find_by_tag(self, tag):
return self.find_by_xpath('//%s' % tag)
def find_by_id(self, id_value):
return self.find_by_xpath('//*[@id="%s"][1]' % id_value)
def find_by_name(self, name):
elements = []
index = 0
while True:
try:
control = self._browser.getControl(name=name, index=index)
elements.append(control)
index += 1
except IndexError:
break
return ElementList([ZopeTestBrowserControlElement(element) for element in elements])
def find_link_by_text(self, text):
return self._find_links_by_xpath("//a[text()='%s']" % text)
def find_link_by_href(self, href):
return self._find_links_by_xpath("//a[@href='%s']" % href)
def fill_in(self, name, value):
self.find_by_name(name=name).first._control.value = value
fill = fill_in
def choose(self, name):
control = self._browser.getControl(name=name)
control.value = control.options
check = choose
def uncheck(self, name):
control = self._browser.getControl(name=name)
control.value = []
def attach_file(self, name, file_path):
control = self._browser.getControl(name=name)
content_type, _ = mimetypes.guess_type(file_path)
control.add_file(open(file_path), content_type, None)
def _find_links_by_xpath(self, xpath):
html = lxml.html.fromstring(self.html)
links = html.xpath(xpath)
return ElementList([ZopeTestBrowserLinkElement(link, self._browser) for link in links])
def select(self, name, value):
self.find_by_name(name).first._control.value = [value,]
def _element_is_link(self, element):
return element.tag == 'a'
def _element_is_control(self, element):
return hasattr(element, 'type')
def __init__(self):
self._browser = Browser()
self._last_urls = []
class Administrator():
'''An automated Google Apps administrator.'''
def __init__(self, domain, username, password):
'''Create an administrator, then log in with the given credentials.'''
self.domain = domain
self.browser = Browser()
self.login(username, password)
def login(self, username, password):
'''Log in with the given username and password.'''
self.browser.open('https://www.google.com/a/%s/ServiceLogin' %
self.domain)
form = self.browser.getForm(id='gaia_loginform')
# domain is automatically added
form.getControl(name='Email').value = username
form.getControl(name='Passwd').value = password
form.submit()
def go_to_group(self, group):
'''Open the groups's page.'''
self.browser.open(
'https://www.google.com/a/cpanel/%s/Group?groupId=%s' %
(self.domain, group))
def users_in_group(self, group):
'''Return the set of users in the group.'''
self.go_to_group(group)
# Add all users in the current (first page).
users = emails_in_html(self.browser.contents)
try:
# Go to next pages and add all of them.
while True:
self.browser.getLink(text='Next').click()
users.update(emails_in_html(self.browser.contents))
except LinkNotFoundError:
pass
return users
def add_users_to_group(self, users, group):
'''Add the users to the group.'''
unique_users = set(users)
current_users = self.users_in_group(group)
new_users = unique_users - current_users
self.go_to_group(group)
for chunk in split_users(new_users):
form = self.browser.getForm(id='addmember')
form.getControl(name='members').value = ','.join(chunk)
# Click this button to submit the form
form.getControl(name='add').click()
def remove_users_from_group(self, users, group):
'''Remove the users from the group.'''
unique_users = set(users)
self.go_to_group(group)
tree = html.fromstring(self.browser.contents)
form = self.browser.getForm(id='list')
for hidden_input in tree.cssselect(
'form#list td input[type="hidden"]'):
print 'checking', hidden_input.value
if hidden_input.value in unique_users:
print 'removing', hidden_input.value
checkbox = form.getControl(name=hidden_input.getnext().name)
print checkbox.mech_control
print 'submitting'
form.getControl(name='remove', index=0).click()
def __init__(self, domain, username, password):
'''Create an administrator, then log in with the given credentials.'''
self.domain = domain
self.browser = Browser()
self.login(username, password)
def __init__(self, *args, **kwargs):
kwargs['mech_browser'] = InterceptBrowser()
ZopeTestBrowser.__init__(self, *args, **kwargs)
def fetch_data(self):
url, username, password = self._url, self._username, self._password
login_url = '/tutos/php/bookinginserter/login.php'
showteamhours_url = '/tutos/php/bookinginserter/showteamhours.php'
browser = Browser(url)
browser.getControl(name='username').value = username
browser.getControl(name='password').value = password
browser.getForm(action=login_url).submit()
browser.getLink(url='showteamhours.php').click()
start_date_value = browser.getControl(name='startdate').options[0]
browser.getControl(name='startdate').value = [start_date_value]
end_date_value = browser.getControl(name='enddate').options[-1]
browser.getControl(name='enddate').value = [end_date_value]
browser.getForm(action=showteamhours_url).submit()
self._from_date = start_date_value
self._to_date = end_date_value
self._data = browser.contents
def __init__(self, user_agent=None):
mech_browser = self._get_mech_browser(user_agent)
self._browser = Browser(mech_browser=mech_browser)
self._cookie_manager = CookieManager(self._browser.cookies)
self._last_urls = []
class ZopeTestBrowser(ElementPresentMixIn, DriverAPI):
driver_name = "zope.testbrowser"
def __init__(self, wait_time=2):
self.wait_time = wait_time
self._browser = Browser()
self._cookie_manager = CookieManager(self._browser)
self._last_urls = []
self.links = FindLinks(self)
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
pass
def visit(self, url):
self._browser.open(url)
def back(self):
self._last_urls.insert(0, self.url)
self._browser.goBack()
def forward(self):
try:
self.visit(self._last_urls.pop())
except IndexError:
pass
def reload(self):
self._browser.reload()
def quit(self):
pass
@property
def htmltree(self):
try:
html = self.html.decode("utf-8")
except AttributeError:
html = self.html
return lxml.html.fromstring(html)
@property
def title(self):
return self._browser.title
@property
def html(self):
return self._browser.contents
@property
def url(self):
return self._browser.url
def find_option_by_value(self, value):
html = self.htmltree
element = html.xpath('//option[@value="%s"]' % value)[0]
control = self._browser.getControl(element.text)
return ElementList([ZopeTestBrowserOptionElement(control, self)],
find_by="value",
query=value)
def find_option_by_text(self, text):
html = self.htmltree
element = html.xpath('//option[normalize-space(text())="%s"]' %
text)[0]
control = self._browser.getControl(element.text)
return ElementList([ZopeTestBrowserOptionElement(control, self)],
find_by="text",
query=text)
def find_by_css(self, selector):
xpath = CSSSelector(selector).path
return self.find_by_xpath(xpath,
original_find="css",
original_query=selector)
def get_control(self, xpath_element):
return xpath_element
def find_by_xpath(self, xpath, original_find=None, original_query=None):
html = self.htmltree
elements = []
for xpath_element in html.xpath(xpath):
if self._element_is_link(xpath_element):
return self._find_links_by_xpath(xpath)
elif self._element_is_control(
xpath_element) and xpath_element.name:
return self.find_by_name(xpath_element.name)
else:
elements.append(self.get_control(xpath_element))
find_by = original_find or "xpath"
query = original_query or xpath
return ElementList(
[ZopeTestBrowserElement(element, self) for element in elements],
find_by=find_by,
query=query,
)
def find_by_tag(self, tag):
return self.find_by_xpath("//%s" % tag,
original_find="tag",
original_query=tag)
def find_by_value(self, value):
elem = self.find_by_xpath('//*[@value="%s"]' % value,
original_find="value",
original_query=value)
if elem:
return elem
return self.find_by_xpath('//*[.="%s"]' % value)
def find_by_text(self, text):
xpath_str = _concat_xpath_from_str(text)
return self.find_by_xpath(
xpath_str,
original_find="text",
original_query=text,
)
def find_by_id(self, id_value):
return self.find_by_xpath(
'//*[@id="%s"][1]' % id_value,
original_find="id",
original_query=id_value,
)
def find_by_name(self, name):
elements = []
index = 0
while True:
try:
control = self._browser.getControl(name=name, index=index)
elements.append(control)
index += 1
except LookupError:
break
except NotImplementedError:
break
return ElementList(
[
ZopeTestBrowserControlElement(element, self)
for element in elements
],
find_by="name",
query=name,
)
def find_link_by_text(self, text):
return self._find_links_by_xpath("//a[text()='%s']" % text)
def find_link_by_href(self, href):
return self._find_links_by_xpath("//a[@href='%s']" % href)
def find_link_by_partial_href(self, partial_href):
return self._find_links_by_xpath("//a[contains(@href, '%s')]" %
partial_href)
def find_link_by_partial_text(self, partial_text):
return self._find_links_by_xpath(
"//a[contains(normalize-space(.), '%s')]" % partial_text)
def fill(self, name, value):
self.find_by_name(name=name).first._control.value = value
def fill_form(self, field_values, form_id=None, name=None):
form = self._browser
if name or form_id:
form = self._browser.getForm(name=name, id=form_id)
for name, value in field_values.items():
control = form.getControl(name=name)
if control.type == "checkbox":
if value:
control.value = control.options
else:
control.value = []
elif control.type == "radio":
control.value = [
option for option in control.options if option == value
]
elif control.type == "select":
control.value = [value]
else:
control.value = value
def choose(self, name, value):
control = self._browser.getControl(name=name)
control.value = [
option for option in control.options if option == value
]
def check(self, name):
control = self._browser.getControl(name=name)
control.value = control.options
def uncheck(self, name):
control = self._browser.getControl(name=name)
control.value = []
def attach_file(self, name, file_path):
filename = file_path.split("/")[-1]
control = self._browser.getControl(name=name)
content_type, _ = mimetypes.guess_type(file_path)
with open(file_path, 'rb') as f:
control.add_file(f, content_type, filename)
def _find_links_by_xpath(self, xpath):
html = self.htmltree
links = html.xpath(xpath)
return ElementList(
[ZopeTestBrowserLinkElement(link, self) for link in links],
find_by="xpath",
query=xpath,
)
def select(self, name, value):
self.find_by_name(name).first._control.value = [value]
def is_text_present(self, text, wait_time=None):
wait_time = wait_time or self.wait_time
end_time = time.time() + wait_time
while time.time() < end_time:
if self._is_text_present(text):
return True
return False
def _is_text_present(self, text):
try:
body = self.find_by_tag("body").first
return text in body.text
except ElementDoesNotExist:
# This exception will be thrown if the body tag isn't present
# This has occasionally been observed. Assume that the
# page isn't fully loaded yet
return False
def is_text_not_present(self, text, wait_time=None):
wait_time = wait_time or self.wait_time
end_time = time.time() + wait_time
while time.time() < end_time:
if not self._is_text_present(text):
return True
return False
def _element_is_link(self, element):
return element.tag == "a"
def _element_is_control(self, element):
return hasattr(element, "type")
@property
def cookies(self):
return self._cookie_manager
class ZopeTestBrowser(DriverAPI):
driver_name = "zope.testbrowser"
def __init__(self):
self._browser = Browser()
self._cookie_manager = CookieManager(self._browser.cookies)
self._last_urls = []
def visit(self, url):
self._browser.open(url)
def back(self):
self._last_urls.insert(0, self.url)
self._browser.goBack()
def forward(self):
try:
self.visit(self._last_urls.pop())
except IndexError:
pass
def reload(self):
self._browser.reload()
def quit(self):
pass
@property
def title(self):
return self._browser.title
@property
def html(self):
return self._browser.contents
@property
def url(self):
return self._browser.url
def find_option_by_value(self, value):
html = lxml.html.fromstring(self.html)
element = html.xpath('//option[@value="%s"]' % value)[0]
control = self._browser.getControl(element.text)
return ElementList([ZopeTestBrowserOptionElement(control, self)], find_by="value", query=value)
def find_option_by_text(self, text):
html = lxml.html.fromstring(self.html)
element = html.xpath('//option[normalize-space(text())="%s"]' % text)[0]
control = self._browser.getControl(element.text)
return ElementList([ZopeTestBrowserOptionElement(control, self)], find_by="text", query=text)
def find_by_css(self, selector):
xpath = CSSSelector(selector).path
return self.find_by_xpath(xpath, original_find="css", original_selector=selector)
def find_by_xpath(self, xpath, original_find=None, original_selector=None):
html = lxml.html.fromstring(self.html)
elements = []
for xpath_element in html.xpath(xpath):
if self._element_is_link(xpath_element):
return self.find_link_by_text(xpath_element.text)
elif self._element_is_control(xpath_element):
return self.find_by_name(xpath_element.name)
else:
elements.append(xpath_element)
find_by = original_find or "xpath"
query = original_selector or xpath
return ElementList([ZopeTestBrowserElement(element, self) for element in elements], find_by=find_by, query=query)
def find_by_tag(self, tag):
return self.find_by_xpath('//%s' % tag, original_find="tag", original_selector=tag)
def find_by_value(self, value):
return self.find_by_xpath('//*[@value="%s"]' % value, original_find="value", original_selector=value)
def find_by_id(self, id_value):
return self.find_by_xpath('//*[@id="%s"][1]' % id_value, original_find="id", original_selector=id_value)
def find_by_name(self, name):
elements = []
index = 0
while True:
try:
control = self._browser.getControl(name=name, index=index)
elements.append(control)
index += 1
except IndexError:
break
return ElementList([ZopeTestBrowserControlElement(element, self) for element in elements], find_by="name", query=name)
def find_link_by_text(self, text):
return self._find_links_by_xpath("//a[text()='%s']" % text)
def find_link_by_href(self, href):
return self._find_links_by_xpath("//a[@href='%s']" % href)
def find_link_by_partial_href(self, partial_href):
return self._find_links_by_xpath("//a[contains(@href, '%s')]" % partial_href)
def find_link_by_partial_text(self, partial_text):
return self._find_links_by_xpath("//a[contains(text(), '%s')]" % partial_text)
def fill(self, name, value):
self.find_by_name(name=name).first._control.value = value
def fill_form(self, field_values):
for name, value in field_values.items():
element = self.find_by_name(name)
control = element.first._control
if control.type == 'text':
control.value = value
elif control.type == 'checkbox':
if value:
control.value = control.options
else:
control.value = []
elif control.type == 'radio':
control.value = [option for option in control.options if option == value]
elif control.type == 'select':
control.value = [value]
def choose(self, name, value):
control = self._browser.getControl(name=name)
control.value = [option for option in control.options if option == value]
def check(self, name):
control = self._browser.getControl(name=name)
control.value = control.options
def uncheck(self, name):
control = self._browser.getControl(name=name)
control.value = []
def attach_file(self, name, file_path):
control = self._browser.getControl(name=name)
content_type, _ = mimetypes.guess_type(file_path)
control.add_file(open(file_path), content_type, None)
def _find_links_by_xpath(self, xpath):
html = lxml.html.fromstring(self.html)
links = html.xpath(xpath)
return ElementList([ZopeTestBrowserLinkElement(link, self) for link in links], find_by="xpath", query=xpath)
def select(self, name, value):
self.find_by_name(name).first._control.value = [value]
def _element_is_link(self, element):
return element.tag == 'a'
def _element_is_control(self, element):
return hasattr(element, 'type')
@property
def cookies(self):
return self._cookie_manager
class ZopeTestBrowser(DriverAPI):
driver_name = "zope.testbrowser"
def __init__(self, user_agent=None):
mech_browser = self._get_mech_browser(user_agent)
self._browser = Browser(mech_browser=mech_browser)
self._cookie_manager = CookieManager(self._browser.cookies)
self._last_urls = []
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
pass
def visit(self, url):
self._browser.open(url)
def back(self):
self._last_urls.insert(0, self.url)
self._browser.goBack()
def forward(self):
try:
self.visit(self._last_urls.pop())
except IndexError:
pass
def reload(self):
self._browser.reload()
def quit(self):
pass
@property
def title(self):
return self._browser.title
@property
def html(self):
return self._browser.contents
@property
def url(self):
return self._browser.url
def find_option_by_value(self, value):
html = lxml.html.fromstring(self.html)
element = html.xpath('//option[@value="%s"]' % value)[0]
control = self._browser.getControl(element.text)
return ElementList([ZopeTestBrowserOptionElement(control, self)], find_by="value", query=value)
def find_option_by_text(self, text):
html = lxml.html.fromstring(self.html)
element = html.xpath('//option[normalize-space(text())="%s"]' % text)[0]
control = self._browser.getControl(element.text)
return ElementList([ZopeTestBrowserOptionElement(control, self)], find_by="text", query=text)
def find_by_css(self, selector):
xpath = CSSSelector(selector).path
return self.find_by_xpath(xpath, original_find="css", original_selector=selector)
def find_by_xpath(self, xpath, original_find=None, original_selector=None):
html = lxml.html.fromstring(self.html)
elements = []
for xpath_element in html.xpath(xpath):
if self._element_is_link(xpath_element):
return self._find_links_by_xpath(xpath)
elif self._element_is_control(xpath_element):
return self.find_by_name(xpath_element.name)
else:
elements.append(xpath_element)
find_by = original_find or "xpath"
query = original_selector or xpath
return ElementList([ZopeTestBrowserElement(element, self) for element in elements], find_by=find_by, query=query)
def find_by_tag(self, tag):
return self.find_by_xpath('//%s' % tag, original_find="tag", original_selector=tag)
def find_by_value(self, value):
return self.find_by_xpath('//*[@value="%s"]' % value, original_find="value", original_selector=value)
def find_by_id(self, id_value):
return self.find_by_xpath('//*[@id="%s"][1]' % id_value, original_find="id", original_selector=id_value)
def find_by_name(self, name):
elements = []
index = 0
while True:
try:
control = self._browser.getControl(name=name, index=index)
elements.append(control)
index += 1
except IndexError:
break
return ElementList([ZopeTestBrowserControlElement(element, self) for element in elements], find_by="name", query=name)
def find_link_by_text(self, text):
return self._find_links_by_xpath("//a[text()='%s']" % text)
def find_link_by_href(self, href):
return self._find_links_by_xpath("//a[@href='%s']" % href)
def find_link_by_partial_href(self, partial_href):
return self._find_links_by_xpath("//a[contains(@href, '%s')]" % partial_href)
def find_link_by_partial_text(self, partial_text):
return self._find_links_by_xpath("//a[contains(text(), '%s')]" % partial_text)
def fill(self, name, value):
self.find_by_name(name=name).first._control.value = value
def fill_form(self, field_values):
for name, value in field_values.items():
element = self.find_by_name(name)
control = element.first._control
if control.type in ['text', 'textarea']:
control.value = value
elif control.type == 'checkbox':
if value:
control.value = control.options
else:
control.value = []
elif control.type == 'radio':
control.value = [option for option in control.options if option == value]
elif control.type == 'select':
control.value = [value]
def choose(self, name, value):
control = self._browser.getControl(name=name)
control.value = [option for option in control.options if option == value]
def check(self, name):
control = self._browser.getControl(name=name)
control.value = control.options
def uncheck(self, name):
control = self._browser.getControl(name=name)
control.value = []
def attach_file(self, name, file_path):
filename = file_path.split('/')[-1]
control = self._browser.getControl(name=name)
content_type, _ = mimetypes.guess_type(file_path)
control.add_file(open(file_path), content_type, filename)
def _find_links_by_xpath(self, xpath):
html = lxml.html.fromstring(self.html)
links = html.xpath(xpath)
return ElementList([ZopeTestBrowserLinkElement(link, self) for link in links], find_by="xpath", query=xpath)
def select(self, name, value):
self.find_by_name(name).first._control.value = [value]
def is_text_present(self, text, wait_time=None):
wait_time = wait_time or 2
end_time = time.time() + wait_time
while time.time() < end_time:
if self._is_text_present(text):
return True
return False
def _is_text_present(self, text):
try:
body = self.find_by_tag('body').first
return text in body.text
except ElementDoesNotExist:
# This exception will be thrown if the body tag isn't present
# This has occasionally been observed. Assume that the
# page isn't fully loaded yet
return False
def is_text_not_present(self, text, wait_time=None):
wait_time = wait_time or 2
end_time = time.time() + wait_time
while time.time() < end_time:
if not self._is_text_present(text):
return True
return False
def _element_is_link(self, element):
return element.tag == 'a'
def _element_is_control(self, element):
return hasattr(element, 'type')
def _get_mech_browser(self, user_agent):
mech_browser = mechanize.Browser()
if user_agent is not None:
mech_browser.addheaders = [("User-agent", user_agent), ]
return mech_browser
@property
def cookies(self):
return self._cookie_manager
#!usr/bin/env python
from zope.testbrowser.browser import Browser
from zope.testbrowser import interfaces
from zope.interface.verify import verifyObject
import urllib
browser = Browser()
browser.raiseHttpErrors = False
#browser.handleErrors = False
print browser
try:
browser.open("https://750words.com/auth", )
except ValueError:
pass
browser.url
browser.headers
email = '*****@*****.**'
password = '******'
signIn = browser.getForm(id='signin_form')
emailEnter = signIn.getControl(name='person[email_address]')
passwordEnter = signIn.getControl(name='person[password]')
emailEnter.value = email
passwordEnter.value = password
signIn.submit()
def make_one(self, *args):
from zope.testbrowser.browser import Browser
return Browser(*args)
class ZopeTestBrowser(DriverAPI):
driver_name = "zope.testbrowser"
def __init__(self):
self._browser = Browser()
self._cookie_manager = CookieManager(self._browser.cookies)
self._last_urls = []
def visit(self, url):
self._browser.open(url)
def back(self):
self._last_urls.insert(0, self.url)
self._browser.goBack()
def forward(self):
try:
self.visit(self._last_urls.pop())
except IndexError:
pass
def reload(self):
self._browser.reload()
def quit(self):
pass
@property
def title(self):
return self._browser.title
@property
def html(self):
return self._browser.contents
@property
def url(self):
return self._browser.url
def find_option_by_value(self, value):
html = lxml.html.fromstring(self.html)
element = html.xpath('//option[@value="%s"]' % value)[0]
control = self._browser.getControl(element.text)
return ElementList([ZopeTestBrowserOptionElement(control, self)], find_by="value", query=value)
def find_option_by_text(self, text):
html = lxml.html.fromstring(self.html)
element = html.xpath('//option[normalize-space(text())="%s"]' % text)[0]
control = self._browser.getControl(element.text)
return ElementList([ZopeTestBrowserOptionElement(control, self)], find_by="text", query=text)
def find_by_css(self, selector):
xpath = CSSSelector(selector).path
return self.find_by_xpath(xpath, original_find="css", original_selector=selector)
def find_by_xpath(self, xpath, original_find=None, original_selector=None):
html = lxml.html.fromstring(self.html)
elements = []
for xpath_element in html.xpath(xpath):
if self._element_is_link(xpath_element):
return self.find_link_by_text(xpath_element.text)
elif self._element_is_control(xpath_element):
return self.find_by_name(xpath_element.name)
else:
elements.append(xpath_element)
find_by = original_find or "xpath"
query = original_selector or xpath
return ElementList(
[ZopeTestBrowserElement(element, self) for element in elements], find_by=find_by, query=query
)
def find_by_tag(self, tag):
return self.find_by_xpath("//%s" % tag, original_find="tag", original_selector=tag)
def find_by_value(self, value):
return self.find_by_xpath('//*[@value="%s"]' % value, original_find="value", original_selector=value)
def find_by_id(self, id_value):
return self.find_by_xpath('//*[@id="%s"][1]' % id_value, original_find="id", original_selector=id_value)
def find_by_name(self, name):
elements = []
index = 0
while True:
try:
control = self._browser.getControl(name=name, index=index)
elements.append(control)
index += 1
except IndexError:
break
return ElementList(
[ZopeTestBrowserControlElement(element, self) for element in elements], find_by="name", query=name
)
def find_link_by_text(self, text):
return self._find_links_by_xpath("//a[text()='%s']" % text)
def find_link_by_href(self, href):
return self._find_links_by_xpath("//a[@href='%s']" % href)
def find_link_by_partial_href(self, partial_href):
return self._find_links_by_xpath("//a[contains(@href, '%s')]" % partial_href)
def find_link_by_partial_text(self, partial_text):
return self._find_links_by_xpath("//a[contains(text(), '%s')]" % partial_text)
def fill(self, name, value):
self.find_by_name(name=name).first._control.value = value
def fill_form(self, field_values):
for name, value in field_values.items():
element = self.find_by_name(name)
control = element.first._control
if control.type == "text":
control.value = value
elif control.type == "checkbox":
if value:
control.value = control.options
else:
control.value = []
elif control.type == "radio":
control.value = [option for option in control.options if option == value]
elif control.type == "select":
control.value = [value]
def choose(self, name, value):
control = self._browser.getControl(name=name)
control.value = [option for option in control.options if option == value]
def check(self, name):
control = self._browser.getControl(name=name)
control.value = control.options
def uncheck(self, name):
control = self._browser.getControl(name=name)
control.value = []
def attach_file(self, name, file_path):
control = self._browser.getControl(name=name)
content_type, _ = mimetypes.guess_type(file_path)
control.add_file(open(file_path), content_type, None)
def _find_links_by_xpath(self, xpath):
html = lxml.html.fromstring(self.html)
links = html.xpath(xpath)
return ElementList([ZopeTestBrowserLinkElement(link, self) for link in links], find_by="xpath", query=xpath)
def select(self, name, value):
self.find_by_name(name).first._control.value = [value]
def _element_is_link(self, element):
return element.tag == "a"
def _element_is_control(self, element):
return hasattr(element, "type")
@property
def cookies(self):
return self._cookie_manager
def open(self,uri,include_server=True):
if include_server:
uri = server+uri
return Browser.open(self,uri)
def __init__(self):
self._browser = Browser()
self._cookie_manager = CookieManager(self._browser.cookies)
self._last_urls = []