def getLink(self, url=None): if not url: return # ///////////////////////////////////////////////////////////////////////////// # Scenario 1: do a simple search for all web server logs # ///////////////////////////////////////////////////////////////////////////// # start search browser = Browser() # Get URL # # Option A: return all of the matched events browser.open(url) ctrl = browser.getControl(name='') ctrl.value = '' ctrl = browser.getControl(name='') ctrl.value = '' ctrl = browser.getControl(name='submitButtonName') ctrl.click() print browser.contents
def login(self): br = Browser() br.mech_browser.set_handle_robots(False) br.open(self.starturl) br.getControl('Username:'******'Password:'******'Login').click() return br
class TestMechRepr(unittest.TestCase): """Testing ..browser.*.mechRepr().""" def setUp(self): super(TestMechRepr, self).setUp() app = TestApp() app.set_next_response(u'''\ <html> <body> <form> <input name="inp1" type="text" value="Täkst" /> <select name="sel1"> <option value="op">Türn</option> </select> <input name="check1" type="checkbox" value="šêlėçtèd" /> <input name="mail1" type="email" value="*****@*****.**" /> <input name="sub1" type="submit" value="Yës" /> </form> </body> </html>'''.encode('utf-8')) self.browser = Browser(wsgi_app=app) self.browser.open('https://localhost') def test_TextControl_has_str_mechRepr(self): mech_repr = self.browser.getControl(name='inp1').mechRepr() self.assertIsInstance(mech_repr, str) self.assertEqual(mech_repr, '<TextControl(inp1=Täkst)>') def test_ItemControl_has_str_mechRepr(self): option = self.browser.getControl(name='sel1').getControl(value="op") mech_repr = option.mechRepr() self.assertIsInstance(mech_repr, str) self.assertEqual( mech_repr, "<Item name='op' id=None contents='Türn' value='op'" " label='Türn'>") def test_CheckboxListControl_has_str_mechRepr(self): from ..browser import CheckboxListControl ctrl = self.browser.getControl(name='check1') self.assertIsInstance(ctrl, CheckboxListControl) mech_repr = ctrl.mechRepr() self.assertIsInstance(mech_repr, str) self.assertEqual(mech_repr, '<SelectControl(check1=[*, ambiguous])>') def test_Control_for_type_email_has_mechRepr(self): option = self.browser.getControl(name='mail1') mech_repr = option.mechRepr() self.assertIsInstance(mech_repr, str) self.assertEqual(mech_repr, "<EMailControl([email protected])>") def test_SubmitControl_has_str_mechRepr(self): mech_repr = self.browser.getControl(name='sub1').mechRepr() self.assertIsInstance(mech_repr, str) self.assertEqual(mech_repr, '<SubmitControl(sub1=Yës)>')
def _getPadsPage(self): username, password, url = self.getUserData() # login browser = Browser() browser.open('%s/ep/account/sign-in' % self.trail_url(url)) browser.getControl(name='email').value = username browser.getControl(name='password').value = password browser.getForm(id='signin-form').submit() # open pads table page browser.getLink('Pads').click() return safe_unicode(browser.contents)
def getBrowser(self, username=TEST_USER_NAME, password=TEST_USER_PASSWORD, loggedIn=True): # Instantiate and return a testbrowser for convenience browser = Browser(self.portal) browser.addHeader("Accept-Language", "en-US") browser.handleErrors = False if loggedIn: browser.open(self.portal.absolute_url()) browser.getControl("Login Name").value = username browser.getControl("Password").value = password browser.getControl("Log in").click() self.assertTrue("You are now logged in" in browser.contents) return browser
def fetch_data(self): url, username, password = self._url, self._username, self._password login_url = '/tutos/php/bookinginserter/login.php' showteamhours_url = '/tutos/php/bookinginserter/showteamhours.php' browser = Browser(url) browser.getControl(name='username').value = username browser.getControl(name='password').value = password browser.getForm(action=login_url).submit() browser.getLink(url='showteamhours.php').click() start_date_value = browser.getControl(name='startdate').options[0] browser.getControl(name='startdate').value = [start_date_value] end_date_value = browser.getControl(name='enddate').options[-1] browser.getControl(name='enddate').value = [end_date_value] browser.getForm(action=showteamhours_url).submit() self._from_date = start_date_value self._to_date = end_date_value self._data = browser.contents
def fetch_data(self): url, username, password = self._url, self._username, self._password login_url = '/tutos/php/bookinginserter/login.php' showteamhours_url = '/tutos/php/bookinginserter/showteamhours.php' browser = Browser(url) browser.getControl(name='username').value = username browser.getControl(name='password').value = password browser.getForm(action=login_url).submit() browser.getLink(url='showteamhours.php').click() start_date_value = browser.getControl(name='startdate').options[0] browser.getControl(name='startdate').value = [start_date_value] end_date_value = browser.getControl(name='enddate').options[-1] browser.getControl(name='enddate').value = [end_date_value] browser.getForm(action=showteamhours_url).submit() self._from_date = start_date_value self._to_date = end_date_value self._data = browser.contents
def setUp(self): super(TestDisplayValue, self).setUp() app = TestApp() app.set_next_response(b'''\ <html> <body> <form> <select name="sel1"> <option value="op">Turn</option> <option value="alt">Alternative</option> </select> </form> </body> </html>''') browser = Browser(wsgi_app=app) browser.open('https://localhost') self.control = browser.getControl(name='sel1')
class ZopeTestBrowser(DriverAPI): driver_name = "zope.testbrowser" def __init__(self, user_agent=None): mech_browser = self._get_mech_browser(user_agent) self._browser = Browser(mech_browser=mech_browser) self._cookie_manager = CookieManager(self._browser.cookies) self._last_urls = [] def __enter__(self): return self def __exit__(self, exc_type, exc_value, traceback): pass def visit(self, url): self._browser.open(url) def back(self): self._last_urls.insert(0, self.url) self._browser.goBack() def forward(self): try: self.visit(self._last_urls.pop()) except IndexError: pass def reload(self): self._browser.reload() def quit(self): pass @property def title(self): return self._browser.title @property def html(self): return self._browser.contents @property def url(self): return self._browser.url def find_option_by_value(self, value): html = lxml.html.fromstring(self.html) element = html.xpath('//option[@value="%s"]' % value)[0] control = self._browser.getControl(element.text) return ElementList([ZopeTestBrowserOptionElement(control, self)], find_by="value", query=value) def find_option_by_text(self, text): html = lxml.html.fromstring(self.html) element = html.xpath('//option[normalize-space(text())="%s"]' % text)[0] control = self._browser.getControl(element.text) return ElementList([ZopeTestBrowserOptionElement(control, self)], find_by="text", query=text) def find_by_css(self, selector): xpath = CSSSelector(selector).path return self.find_by_xpath(xpath, original_find="css", original_selector=selector) def find_by_xpath(self, xpath, original_find=None, original_selector=None): html = lxml.html.fromstring(self.html) elements = [] for xpath_element in html.xpath(xpath): if self._element_is_link(xpath_element): return self._find_links_by_xpath(xpath) elif self._element_is_control(xpath_element): return self.find_by_name(xpath_element.name) else: elements.append(xpath_element) find_by = original_find or "xpath" query = original_selector or xpath return ElementList([ZopeTestBrowserElement(element, self) for element in elements], find_by=find_by, query=query) def find_by_tag(self, tag): return self.find_by_xpath('//%s' % tag, original_find="tag", original_selector=tag) def find_by_value(self, value): return self.find_by_xpath('//*[@value="%s"]' % value, original_find="value", original_selector=value) def find_by_id(self, id_value): return self.find_by_xpath('//*[@id="%s"][1]' % id_value, original_find="id", original_selector=id_value) def find_by_name(self, name): elements = [] index = 0 while True: try: control = self._browser.getControl(name=name, index=index) elements.append(control) index += 1 except IndexError: break return ElementList([ZopeTestBrowserControlElement(element, self) for element in elements], find_by="name", query=name) def find_link_by_text(self, text): return self._find_links_by_xpath("//a[text()='%s']" % text) def find_link_by_href(self, href): return self._find_links_by_xpath("//a[@href='%s']" % href) def find_link_by_partial_href(self, partial_href): return self._find_links_by_xpath("//a[contains(@href, '%s')]" % partial_href) def find_link_by_partial_text(self, partial_text): return self._find_links_by_xpath("//a[contains(text(), '%s')]" % partial_text) def fill(self, name, value): self.find_by_name(name=name).first._control.value = value def fill_form(self, field_values): for name, value in field_values.items(): element = self.find_by_name(name) control = element.first._control if control.type in ['text', 'textarea']: control.value = value elif control.type == 'checkbox': if value: control.value = control.options else: control.value = [] elif control.type == 'radio': control.value = [option for option in control.options if option == value] elif control.type == 'select': control.value = [value] def choose(self, name, value): control = self._browser.getControl(name=name) control.value = [option for option in control.options if option == value] def check(self, name): control = self._browser.getControl(name=name) control.value = control.options def uncheck(self, name): control = self._browser.getControl(name=name) control.value = [] def attach_file(self, name, file_path): filename = file_path.split('/')[-1] control = self._browser.getControl(name=name) content_type, _ = mimetypes.guess_type(file_path) control.add_file(open(file_path), content_type, filename) def _find_links_by_xpath(self, xpath): html = lxml.html.fromstring(self.html) links = html.xpath(xpath) return ElementList([ZopeTestBrowserLinkElement(link, self) for link in links], find_by="xpath", query=xpath) def select(self, name, value): self.find_by_name(name).first._control.value = [value] def is_text_present(self, text, wait_time=None): wait_time = wait_time or 2 end_time = time.time() + wait_time while time.time() < end_time: if self._is_text_present(text): return True return False def _is_text_present(self, text): try: body = self.find_by_tag('body').first return text in body.text except ElementDoesNotExist: # This exception will be thrown if the body tag isn't present # This has occasionally been observed. Assume that the # page isn't fully loaded yet return False def is_text_not_present(self, text, wait_time=None): wait_time = wait_time or 2 end_time = time.time() + wait_time while time.time() < end_time: if not self._is_text_present(text): return True return False def _element_is_link(self, element): return element.tag == 'a' def _element_is_control(self, element): return hasattr(element, 'type') def _get_mech_browser(self, user_agent): mech_browser = mechanize.Browser() if user_agent is not None: mech_browser.addheaders = [("User-agent", user_agent), ] return mech_browser @property def cookies(self): return self._cookie_manager
class ZopeTestBrowser(ElementPresentMixIn, DriverAPI): driver_name = "zope.testbrowser" def __init__(self, wait_time=2): self.wait_time = wait_time self._browser = Browser() self._cookie_manager = CookieManager(self._browser) self._last_urls = [] self.links = FindLinks(self) def __enter__(self): return self def __exit__(self, exc_type, exc_value, traceback): pass def visit(self, url): self._browser.open(url) def back(self): self._last_urls.insert(0, self.url) self._browser.goBack() def forward(self): try: self.visit(self._last_urls.pop()) except IndexError: pass def reload(self): self._browser.reload() def quit(self): pass @property def htmltree(self): try: html = self.html.decode("utf-8") except AttributeError: html = self.html return lxml.html.fromstring(html) @property def title(self): return self._browser.title @property def html(self): return self._browser.contents @property def url(self): return self._browser.url def find_option_by_value(self, value): html = self.htmltree element = html.xpath('//option[@value="%s"]' % value)[0] control = self._browser.getControl(element.text) return ElementList([ZopeTestBrowserOptionElement(control, self)], find_by="value", query=value) def find_option_by_text(self, text): html = self.htmltree element = html.xpath('//option[normalize-space(text())="%s"]' % text)[0] control = self._browser.getControl(element.text) return ElementList([ZopeTestBrowserOptionElement(control, self)], find_by="text", query=text) def find_by_css(self, selector): xpath = CSSSelector(selector).path return self.find_by_xpath(xpath, original_find="css", original_query=selector) def get_control(self, xpath_element): return xpath_element def find_by_xpath(self, xpath, original_find=None, original_query=None): html = self.htmltree elements = [] for xpath_element in html.xpath(xpath): if self._element_is_link(xpath_element): return self._find_links_by_xpath(xpath) elif self._element_is_control( xpath_element) and xpath_element.name: return self.find_by_name(xpath_element.name) else: elements.append(self.get_control(xpath_element)) find_by = original_find or "xpath" query = original_query or xpath return ElementList( [ZopeTestBrowserElement(element, self) for element in elements], find_by=find_by, query=query, ) def find_by_tag(self, tag): return self.find_by_xpath("//%s" % tag, original_find="tag", original_query=tag) def find_by_value(self, value): elem = self.find_by_xpath('//*[@value="%s"]' % value, original_find="value", original_query=value) if elem: return elem return self.find_by_xpath('//*[.="%s"]' % value) def find_by_text(self, text): xpath_str = _concat_xpath_from_str(text) return self.find_by_xpath( xpath_str, original_find="text", original_query=text, ) def find_by_id(self, id_value): return self.find_by_xpath( '//*[@id="%s"][1]' % id_value, original_find="id", original_query=id_value, ) def find_by_name(self, name): elements = [] index = 0 while True: try: control = self._browser.getControl(name=name, index=index) elements.append(control) index += 1 except LookupError: break except NotImplementedError: break return ElementList( [ ZopeTestBrowserControlElement(element, self) for element in elements ], find_by="name", query=name, ) def find_link_by_text(self, text): return self._find_links_by_xpath("//a[text()='%s']" % text) def find_link_by_href(self, href): return self._find_links_by_xpath("//a[@href='%s']" % href) def find_link_by_partial_href(self, partial_href): return self._find_links_by_xpath("//a[contains(@href, '%s')]" % partial_href) def find_link_by_partial_text(self, partial_text): return self._find_links_by_xpath( "//a[contains(normalize-space(.), '%s')]" % partial_text) def fill(self, name, value): self.find_by_name(name=name).first._control.value = value def fill_form(self, field_values, form_id=None, name=None): form = self._browser if name or form_id: form = self._browser.getForm(name=name, id=form_id) for name, value in field_values.items(): control = form.getControl(name=name) if control.type == "checkbox": if value: control.value = control.options else: control.value = [] elif control.type == "radio": control.value = [ option for option in control.options if option == value ] elif control.type == "select": control.value = [value] else: control.value = value def choose(self, name, value): control = self._browser.getControl(name=name) control.value = [ option for option in control.options if option == value ] def check(self, name): control = self._browser.getControl(name=name) control.value = control.options def uncheck(self, name): control = self._browser.getControl(name=name) control.value = [] def attach_file(self, name, file_path): filename = file_path.split("/")[-1] control = self._browser.getControl(name=name) content_type, _ = mimetypes.guess_type(file_path) with open(file_path, 'rb') as f: control.add_file(f, content_type, filename) def _find_links_by_xpath(self, xpath): html = self.htmltree links = html.xpath(xpath) return ElementList( [ZopeTestBrowserLinkElement(link, self) for link in links], find_by="xpath", query=xpath, ) def select(self, name, value): self.find_by_name(name).first._control.value = [value] def is_text_present(self, text, wait_time=None): wait_time = wait_time or self.wait_time end_time = time.time() + wait_time while time.time() < end_time: if self._is_text_present(text): return True return False def _is_text_present(self, text): try: body = self.find_by_tag("body").first return text in body.text except ElementDoesNotExist: # This exception will be thrown if the body tag isn't present # This has occasionally been observed. Assume that the # page isn't fully loaded yet return False def is_text_not_present(self, text, wait_time=None): wait_time = wait_time or self.wait_time end_time = time.time() + wait_time while time.time() < end_time: if not self._is_text_present(text): return True return False def _element_is_link(self, element): return element.tag == "a" def _element_is_control(self, element): return hasattr(element, "type") @property def cookies(self): return self._cookie_manager
class DAInstaller: """ Automatically download DeviceAtlas API bundle and extract. DeviceAtlas Python API files are downloaded from and installed to ${your home folder}/da_distribution. This is to keep proprietary files away from your own (potentially open source) source tree. zc.testbrowser class is used to mimic the real web browser. To install zope.testbrowser, type: easy_install zope.testbrowser You might or might not want to do this, since Zope dependencies are quite heavy. """ def __init__(self, username, password, path=None): """ @param username: DeviceAtlas username @param password: DeviceAtlas password @param path: Path to install API and database files or None to use ~/da_distribution. Path is added to PYTHONPATH. """ try: from zope.testbrowser.browser import Browser except ImportError: raise raise RuntimeError("zope.testbrowser must be installed to automatize DeviceAtlas downloads") self.browser = Browser() if not path: path = os.path.abspath(os.path.join(os.getenv("HOME"), "da_distribution")) self.path = path # Wrap things away from Python namespace self.api_path = os.path.join(path, "da_api") if not os.path.exists(self.path): os.makedirs(self.path) os.makedirs(self.api_path) if not path in sys.path: # Include autodownload folder in Python import sys.path.append(self.path) self.fname = "da.zip" self.json_file = "json.zip" self.raw_json = os.path.join(self.path, "data.json") # Unzipped json database self.username = username self.password = password self.login_url = "http://deviceatlas.com" self.api_url = url="http://deviceatlas.com/getAPI/python" def login(self): """ """ self.browser.open(self.login_url) if "Signed in as" in self.browser.contents: return # already logged in field = self.browser.getControl(name="name") field.value = self.username field = self.browser.getControl(name="pass") field.value = self.password submit = self.browser.getControl("Go") submit.click() if not "http://deviceatlas.com/user" in self.browser.url: raise RuntimeError("Could not log in to deviceatlas.com with username " + self.username) #print self.browser.url #print self.browser.headers #print self.browser.contents def need_download(self): """ @return: True if zip download needed """ if not os.path.exists(os.path.join(self.path, self.fname)): return True return False def need_unzip(self): """ @return: True if zip extract needed """ if not os.path.exists(os.path.join(self.api_path, "api.py")): return True return False def download_zip(self): import urllib2 self.browser.open("http://deviceatlas.com/getAPI/python") assert self.browser.headers["Content-Type"] == "application/zip" # Check that we fetch the correct file assert len(self.browser.contents) > 0, "Could not download da.zip API distribution" o = open(os.path.join(self.path, self.fname), "wb") o.write(self.browser.contents) o.close() def unzip_file_into_dir(self, file, dir, override_name=None): import zipfile if not os.path.exists(dir): os.mkdir(dir) zfobj = zipfile.ZipFile(file) for name in zfobj.namelist(): if name.endswith('/'): os.mkdir(os.path.join(dir, name)) else: if override_name: # Set extracted file name fname = override_name else: fname = name outfile = open(os.path.join(dir, fname), 'wb') outfile.write(zfobj.read(name)) outfile.close() def need_database(self): if not os.path.exists(self.raw_json): return True return False def download_database(self): self.browser.open("http://deviceatlas.com") self.browser.getLink("My account").click() # Assume Community ZIP link self.browser.getLink(url="/zip", index=0).click() # Check that we fetch the correct file assert self.browser.headers["Content-Type"] == "application/zip", "Assume zip content:" + str(self.browser.headers) o = open(os.path.join(self.path, self.json_file), "wb") o.write(self.browser.contents) o.close() def proceed(self): """ Download & unzip DA Python API files if needed. @return: Path to Downloaded DeviceAtlas JSON data file """ if self.need_download(): self.login() self.download_zip() if self.need_unzip(): self.unzip_file_into_dir(os.path.join(self.path, self.fname), self.api_path) # create __init__.py to mark the module f = open(os.path.join(self.api_path, "__init__.py"), "wt") f.write("# This package is automatically generated by DAAPIInstaller") f.close() if self.need_database(): self.login() self.download_database() self.unzip_file_into_dir(os.path.join(self.path, self.json_file), self.path, "data.json") return self.raw_json
class ZopeTestBrowser(DriverAPI): def __init__(self): self._browser = Browser() self._last_urls = [] def visit(self, url): self._browser.open(url) def back(self): self._last_urls.insert(0, self.url) self._browser.goBack() def forward(self): try: self.visit(self._last_urls.pop()) except IndexError: pass def reload(self): self._browser.reload() def quit(self): pass @property def title(self): return self._browser.title @property def html(self): return self._browser.contents @property def url(self): return self._browser.url def find_option_by_value(self, value): html = lxml.html.fromstring(self.html) element = html.xpath('//option[@value="%s"]' % value)[0] control = self._browser.getControl(element.text) return ElementList([ZopeTestBrowserOptionElement(control, self)]) def find_option_by_text(self, text): html = lxml.html.fromstring(self.html) element = html.xpath('//option[normalize-space(text())="%s"]' % text)[0] control = self._browser.getControl(element.text) return ElementList([ZopeTestBrowserOptionElement(control, self)]) def find_by_css(self, selector): xpath = CSSSelector(selector).path return self.find_by_xpath(xpath) find_by_css_selector = warn_deprecated(find_by_css, 'find_by_css_selector') def find_by_xpath(self, xpath): html = lxml.html.fromstring(self.html) elements = [] for xpath_element in html.xpath(xpath): if self._element_is_link(xpath_element): return self.find_link_by_text(xpath_element.text) elif self._element_is_control(xpath_element): return self.find_by_name(xpath_element.name) else: elements.append(xpath_element) return ElementList([ZopeTestBrowserElement(element, self) for element in elements]) def find_by_tag(self, tag): return self.find_by_xpath('//%s' % tag) def find_by_id(self, id_value): return self.find_by_xpath('//*[@id="%s"][1]' % id_value) def find_by_name(self, name): elements = [] index = 0 while True: try: control = self._browser.getControl(name=name, index=index) elements.append(control) index += 1 except IndexError: break return ElementList([ZopeTestBrowserControlElement(element, self) for element in elements]) def find_link_by_text(self, text): return self._find_links_by_xpath("//a[text()='%s']" % text) def find_link_by_href(self, href): return self._find_links_by_xpath("//a[@href='%s']" % href) def fill(self, name, value): self.find_by_name(name=name).first._control.value = value fill_in = warn_deprecated(fill, 'fill_in') def choose(self, name): control = self._browser.getControl(name=name) control.value = control.options check = choose def uncheck(self, name): control = self._browser.getControl(name=name) control.value = [] def attach_file(self, name, file_path): control = self._browser.getControl(name=name) content_type, _ = mimetypes.guess_type(file_path) control.add_file(open(file_path), content_type, None) def _find_links_by_xpath(self, xpath): html = lxml.html.fromstring(self.html) links = html.xpath(xpath) return ElementList([ZopeTestBrowserLinkElement(link, self) for link in links]) def select(self, name, value): self.find_by_name(name).first._control.value = [value,] def _element_is_link(self, element): return element.tag == 'a' def _element_is_control(self, element): return hasattr(element, 'type')
def create_posts(self): self.grab_links() filenames = sorted(self.links.keys()) chunker = Chunker(self.links_per_post) n = 1 if n < self.start_on_post: log.info("Skipping posts %d to %d", n, self.start_on_post) for chunk in chunker(filenames): if n < self.start_on_post: n += 1 continue log.info("Processing post %s of %s", n, self.total_posts) title = self.title_base + ' - %d of %d' % (n, self.total_posts) links = [] for filename in chunk: screenshot_name = filename + '.jpg' screenshot_path = os.path.join(self.screenshots_dir, screenshot_name) image_html = "<em>Missing Image preview for <b>%s</b></em>" % filename if os.path.isfile(screenshot_path): log.info("Uploading screenshot %s", screenshot_name) browser = Browser("http://www.freeporndumpster.com/legacy.php") caption_control = browser.getControl(name='imagename[]', index=0) caption_control.value = self.links[filename]['href'] file_control = browser.getControl(name='images[]', index=0) file_control.add_file( open(screenshot_path), "image/jpeg", screenshot_name ) browser.getControl('upload').click() doc = etree.HTML(browser.contents) image_html = "<em>Missing Image html for <b>%s</b></em>" % filename image_html_match = doc.xpath('//table/tr/td/p[contains(., "website")]/textarea/text()') if image_html_match: image_html = image_html_match[0] else: log.error("Failed to get uploaded image html") else: log.warn("There's no screenshot for filename %s", filename) links.append(( filename, self.links[filename]['size'], self.links[filename]['href'], image_html )) html = HTML_TEMPLATE.render( links=links, initial_text=self.email_initial_text, ending_text=self.email_ending_text ) open(os.path.join(self.output_dir, title+'.txt'), 'w').write(html) n += 1 server = None email = MIMEMultipart('alternative') email['Subject'] = title email['From'] = '*****@*****.**' email['To'] = self.email_recipient email.attach(MIMEText(html, 'html')) log.info("Sending post email") try: try: # Python 2.6 server = smtplib.SMTP(self.smtp_host, self.smtp_port, timeout=60) except: # Python 2.5 server = smtplib.SMTP(self.smtp_host, self.smtp_port) except Exception, err: log.error("There was an error sending the notification email: %s", err) if server is None: log.error("Failed to setup SMTP server") continue security_enabled = self.smtp_use_tls if security_enabled: server.ehlo() if not server.esmtp_features.has_key('starttls'): log.warning("TLS/SSL enabled but server does not support it") else: server.starttls() server.ehlo() if self.smtp_user and self.smtp_pass: try: server.login(self.smtp_user, self.smtp_pass) except smtplib.SMTPHeloError, err: log.error("The server didn't reply properly to the helo " "greeting: %s", err) except smtplib.SMTPAuthenticationError, err: log.error("The server didn't accept the username/password " "combination: %s", err)
class ZopeTestBrowser(DriverAPI): driver_name = "zope.testbrowser" def __init__(self): self._browser = Browser() self._cookie_manager = CookieManager(self._browser.cookies) self._last_urls = [] def visit(self, url): self._browser.open(url) def back(self): self._last_urls.insert(0, self.url) self._browser.goBack() def forward(self): try: self.visit(self._last_urls.pop()) except IndexError: pass def reload(self): self._browser.reload() def quit(self): pass @property def title(self): return self._browser.title @property def html(self): return self._browser.contents @property def url(self): return self._browser.url def find_option_by_value(self, value): html = lxml.html.fromstring(self.html) element = html.xpath('//option[@value="%s"]' % value)[0] control = self._browser.getControl(element.text) return ElementList([ZopeTestBrowserOptionElement(control, self)], find_by="value", query=value) def find_option_by_text(self, text): html = lxml.html.fromstring(self.html) element = html.xpath('//option[normalize-space(text())="%s"]' % text)[0] control = self._browser.getControl(element.text) return ElementList([ZopeTestBrowserOptionElement(control, self)], find_by="text", query=text) def find_by_css(self, selector): xpath = CSSSelector(selector).path return self.find_by_xpath(xpath, original_find="css", original_selector=selector) def find_by_xpath(self, xpath, original_find=None, original_selector=None): html = lxml.html.fromstring(self.html) elements = [] for xpath_element in html.xpath(xpath): if self._element_is_link(xpath_element): return self.find_link_by_text(xpath_element.text) elif self._element_is_control(xpath_element): return self.find_by_name(xpath_element.name) else: elements.append(xpath_element) find_by = original_find or "xpath" query = original_selector or xpath return ElementList([ZopeTestBrowserElement(element, self) for element in elements], find_by=find_by, query=query) def find_by_tag(self, tag): return self.find_by_xpath('//%s' % tag, original_find="tag", original_selector=tag) def find_by_value(self, value): return self.find_by_xpath('//*[@value="%s"]' % value, original_find="value", original_selector=value) def find_by_id(self, id_value): return self.find_by_xpath('//*[@id="%s"][1]' % id_value, original_find="id", original_selector=id_value) def find_by_name(self, name): elements = [] index = 0 while True: try: control = self._browser.getControl(name=name, index=index) elements.append(control) index += 1 except IndexError: break return ElementList([ZopeTestBrowserControlElement(element, self) for element in elements], find_by="name", query=name) def find_link_by_text(self, text): return self._find_links_by_xpath("//a[text()='%s']" % text) def find_link_by_href(self, href): return self._find_links_by_xpath("//a[@href='%s']" % href) def find_link_by_partial_href(self, partial_href): return self._find_links_by_xpath("//a[contains(@href, '%s')]" % partial_href) def find_link_by_partial_text(self, partial_text): return self._find_links_by_xpath("//a[contains(text(), '%s')]" % partial_text) def fill(self, name, value): self.find_by_name(name=name).first._control.value = value def fill_form(self, field_values): for name, value in field_values.items(): element = self.find_by_name(name) control = element.first._control if control.type == 'text': control.value = value elif control.type == 'checkbox': if value: control.value = control.options else: control.value = [] elif control.type == 'radio': control.value = [option for option in control.options if option == value] elif control.type == 'select': control.value = [value] def choose(self, name, value): control = self._browser.getControl(name=name) control.value = [option for option in control.options if option == value] def check(self, name): control = self._browser.getControl(name=name) control.value = control.options def uncheck(self, name): control = self._browser.getControl(name=name) control.value = [] def attach_file(self, name, file_path): control = self._browser.getControl(name=name) content_type, _ = mimetypes.guess_type(file_path) control.add_file(open(file_path), content_type, None) def _find_links_by_xpath(self, xpath): html = lxml.html.fromstring(self.html) links = html.xpath(xpath) return ElementList([ZopeTestBrowserLinkElement(link, self) for link in links], find_by="xpath", query=xpath) def select(self, name, value): self.find_by_name(name).first._control.value = [value] def _element_is_link(self, element): return element.tag == 'a' def _element_is_control(self, element): return hasattr(element, 'type') @property def cookies(self): return self._cookie_manager
class ZopeTestBrowser(ElementPresentMixIn, DriverAPI): driver_name = "zope.testbrowser" def __init__(self, user_agent=None, wait_time=2, ignore_robots=False): self.wait_time = wait_time mech_browser = self._get_mech_browser(user_agent, ignore_robots) self._browser = Browser(mech_browser=mech_browser) self._cookie_manager = CookieManager(self._browser.cookies) self._last_urls = [] def __enter__(self): return self def __exit__(self, exc_type, exc_value, traceback): pass def visit(self, url): self._browser.open(url) def back(self): self._last_urls.insert(0, self.url) self._browser.goBack() def forward(self): try: self.visit(self._last_urls.pop()) except IndexError: pass def reload(self): self._browser.reload() def quit(self): pass @property def htmltree(self): return lxml.html.fromstring(self.html.decode('utf-8')) @property def title(self): return self._browser.title @property def html(self): return self._browser.contents @property def url(self): return self._browser.url def find_option_by_value(self, value): html = self.htmltree element = html.xpath('//option[@value="%s"]' % value)[0] control = self._browser.getControl(element.text) return ElementList([ZopeTestBrowserOptionElement(control, self)], find_by="value", query=value) def find_option_by_text(self, text): html = self.htmltree element = html.xpath('//option[normalize-space(text())="%s"]' % text)[0] control = self._browser.getControl(element.text) return ElementList([ZopeTestBrowserOptionElement(control, self)], find_by="text", query=text) def find_by_css(self, selector): xpath = CSSSelector(selector).path return self.find_by_xpath(xpath, original_find="css", original_selector=selector) def find_by_xpath(self, xpath, original_find=None, original_selector=None): html = self.htmltree elements = [] for xpath_element in html.xpath(xpath): if self._element_is_link(xpath_element): return self._find_links_by_xpath(xpath) elif self._element_is_control(xpath_element): return self.find_by_name(xpath_element.name) else: elements.append(xpath_element) find_by = original_find or "xpath" query = original_selector or xpath return ElementList( [ZopeTestBrowserElement(element, self) for element in elements], find_by=find_by, query=query) def find_by_tag(self, tag): return self.find_by_xpath('//%s' % tag, original_find="tag", original_selector=tag) def find_by_value(self, value): return self.find_by_xpath('//*[@value="%s"]' % value, original_find="value", original_selector=value) def find_by_text(self, text): return self.find_by_xpath('//*[text()="%s"]' % text, original_find="text", original_selector=text) def find_by_id(self, id_value): return self.find_by_xpath('//*[@id="%s"][1]' % id_value, original_find="id", original_selector=id_value) def find_by_name(self, name): elements = [] index = 0 while True: try: control = self._browser.getControl(name=name, index=index) elements.append(control) index += 1 except LookupError: break return ElementList([ ZopeTestBrowserControlElement(element, self) for element in elements ], find_by="name", query=name) def find_link_by_text(self, text): return self._find_links_by_xpath("//a[text()='%s']" % text) def find_link_by_href(self, href): return self._find_links_by_xpath("//a[@href='%s']" % href) def find_link_by_partial_href(self, partial_href): return self._find_links_by_xpath("//a[contains(@href, '%s')]" % partial_href) def find_link_by_partial_text(self, partial_text): return self._find_links_by_xpath( "//a[contains(normalize-space(.), '%s')]" % partial_text) def fill(self, name, value): self.find_by_name(name=name).first._control.value = value def fill_form(self, field_values): for name, value in field_values.items(): element = self.find_by_name(name) control = element.first._control if control.type == 'checkbox': if value: control.value = control.options else: control.value = [] elif control.type == 'radio': control.value = [ option for option in control.options if option == value ] elif control.type == 'select': element.select(value) else: # text, textarea, password, tel control.value = value def choose(self, name, value): control = self._browser.getControl(name=name) control.value = [ option for option in control.options if option == value ] def check(self, name): control = self._browser.getControl(name=name) control.value = control.options def uncheck(self, name): control = self._browser.getControl(name=name) control.value = [] def attach_file(self, name, file_path): filename = file_path.split('/')[-1] control = self._browser.getControl(name=name) content_type, _ = mimetypes.guess_type(file_path) control.add_file(open(file_path), content_type, filename) def _find_links_by_xpath(self, xpath): html = self.htmltree links = html.xpath(xpath) return ElementList( [ZopeTestBrowserLinkElement(link, self) for link in links], find_by="xpath", query=xpath) def select(self, name, value): self.find_by_name(name).first._control.value = [value] def is_text_present(self, text, wait_time=None): wait_time = wait_time or self.wait_time end_time = time.time() + wait_time while time.time() < end_time: if self._is_text_present(text): return True return False def _is_text_present(self, text): try: body = self.find_by_tag('body').first return text in body.text except ElementDoesNotExist: # This exception will be thrown if the body tag isn't present # This has occasionally been observed. Assume that the # page isn't fully loaded yet return False def is_text_not_present(self, text, wait_time=None): wait_time = wait_time or self.wait_time end_time = time.time() + wait_time while time.time() < end_time: if not self._is_text_present(text): return True return False def _element_is_link(self, element): return element.tag == 'a' def _element_is_control(self, element): return hasattr(element, 'type') def _get_mech_browser(self, user_agent, ignore_robots): mech_browser = mechanize.Browser() if user_agent is not None: mech_browser.addheaders = [ ("User-agent", user_agent), ] if ignore_robots: mech_browser.set_handle_robots(False) return mech_browser @property def cookies(self): return self._cookie_manager
class ZopeTestBrowser(DriverAPI): def __init__(self): self._browser = Browser() self._cookie_manager = CookieManager(self._browser.cookies) self._last_urls = [] def visit(self, url): self._browser.open(url) def back(self): self._last_urls.insert(0, self.url) self._browser.goBack() def forward(self): try: self.visit(self._last_urls.pop()) except IndexError: pass def reload(self): self._browser.reload() def quit(self): pass @property def title(self): return self._browser.title @property def html(self): return self._browser.contents @property def url(self): return self._browser.url def find_option_by_value(self, value): html = lxml.html.fromstring(self.html) element = html.xpath('//option[@value="%s"]' % value)[0] control = self._browser.getControl(element.text) return ElementList([ZopeTestBrowserOptionElement(control, self)]) def find_option_by_text(self, text): html = lxml.html.fromstring(self.html) element = html.xpath('//option[normalize-space(text())="%s"]' % text)[0] control = self._browser.getControl(element.text) return ElementList([ZopeTestBrowserOptionElement(control, self)]) def find_by_css(self, selector): xpath = CSSSelector(selector).path return self.find_by_xpath(xpath) find_by_css_selector = warn_deprecated(find_by_css, 'find_by_css_selector') def find_by_xpath(self, xpath): html = lxml.html.fromstring(self.html) elements = [] for xpath_element in html.xpath(xpath): if self._element_is_link(xpath_element): return self.find_link_by_text(xpath_element.text) elif self._element_is_control(xpath_element): return self.find_by_name(xpath_element.name) else: elements.append(xpath_element) return ElementList([ZopeTestBrowserElement(element, self) for element in elements]) def find_by_tag(self, tag): return self.find_by_xpath('//%s' % tag) def find_by_value(self, value): return self.find_by_xpath('//*[@value="%s"]' % value) def find_by_id(self, id_value): return self.find_by_xpath('//*[@id="%s"][1]' % id_value) def find_by_name(self, name): elements = [] index = 0 while True: try: control = self._browser.getControl(name=name, index=index) elements.append(control) index += 1 except IndexError: break return ElementList([ZopeTestBrowserControlElement(element, self) for element in elements]) def find_link_by_text(self, text): return self._find_links_by_xpath("//a[text()='%s']" % text) def find_link_by_href(self, href): return self._find_links_by_xpath("//a[@href='%s']" % href) def find_link_by_partial_text(self, partial_text): return self._find_links_by_xpath("//a[contains(text(), '%s')]" % partial_text) def fill(self, name, value): self.find_by_name(name=name).first._control.value = value fill_in = warn_deprecated(fill, 'fill_in') def choose(self, name, value): control = self._browser.getControl(name=name) control.value = [option for option in control.options if option == value] def check(self, name): control = self._browser.getControl(name=name) control.value = control.options def uncheck(self, name): control = self._browser.getControl(name=name) control.value = [] def attach_file(self, name, file_path): control = self._browser.getControl(name=name) content_type, _ = mimetypes.guess_type(file_path) control.add_file(open(file_path), content_type, None) def _find_links_by_xpath(self, xpath): html = lxml.html.fromstring(self.html) links = html.xpath(xpath) return ElementList([ZopeTestBrowserLinkElement(link, self) for link in links]) def select(self, name, value): self.find_by_name(name).first._control.value = [value] def _element_is_link(self, element): return element.tag == 'a' def _element_is_control(self, element): return hasattr(element, 'type') @property def cookies(self): return self._cookie_manager
class ZopeTestBrowser(DriverAPI): def __init__(self): self._browser = Browser() def visit(self, url): self._browser.open(url) def quit(self): pass @property def title(self): return self._browser.title @property def html(self): return self._browser.contents @property def url(self): return self._browser.url def find_option_by_value(self, value): html = lxml.html.fromstring(self.html) element = html.xpath('//option[@value="%s"]' % value)[0] control = self._browser.getControl(element.text) return ElementList([ZopeTestBrowserOptionElement(control)]) def find_option_by_text(self, text): html = lxml.html.fromstring(self.html) element = html.xpath('//option[normalize-space(text())="%s"]' % text)[0] control = self._browser.getControl(element.text) return ElementList([ZopeTestBrowserOptionElement(control)]) def find_by_css_selector(self, selector): xpath = CSSSelector(selector).path return self.find_by_xpath(xpath) def find_by_xpath(self, xpath): html = lxml.html.fromstring(self.html) elements = [] for xpath_element in html.xpath(xpath): if self._element_is_link(xpath_element): return self.find_link_by_text(xpath_element.text) elif self._element_is_control(xpath_element): return self.find_by_name(xpath_element.name) else: elements.append(xpath_element) return ElementList([ZopeTestBrowserElement(element) for element in elements]) def find_by_tag(self, tag): return self.find_by_xpath('//%s' % tag) def find_by_id(self, id_value): return self.find_by_xpath('//*[@id="%s"][1]' % id_value) def find_by_name(self, name): elements = [] index = 0 while True: try: control = self._browser.getControl(name=name, index=index) elements.append(control) index += 1 except IndexError: break return ElementList([ZopeTestBrowserControlElement(element) for element in elements]) def find_link_by_text(self, text): return self._find_links_by_xpath("//a[text()='%s']" % text) def find_link_by_href(self, href): return self._find_links_by_xpath("//a[@href='%s']" % href) def fill_in(self, name, value): self.find_by_name(name=name).first._control.value = value fill = fill_in def choose(self, name): control = self._browser.getControl(name=name) control.value = control.options check = choose def uncheck(self, name): control = self._browser.getControl(name=name) control.value = [] def attach_file(self, name, file_path): control = self._browser.getControl(name=name) content_type, _ = mimetypes.guess_type(file_path) control.add_file(open(file_path), content_type, None) def _find_links_by_xpath(self, xpath): html = lxml.html.fromstring(self.html) links = html.xpath(xpath) return ElementList([ZopeTestBrowserLinkElement(link, self._browser) for link in links]) def select(self, name, value): self.find_by_name(name).first._control.value = [value,] def _element_is_link(self, element): return element.tag == 'a' def _element_is_control(self, element): return hasattr(element, 'type')
class ZopeTestBrowser(DriverAPI): driver_name = "zope.testbrowser" def __init__(self): self._browser = Browser() self._cookie_manager = CookieManager(self._browser.cookies) self._last_urls = [] def visit(self, url): self._browser.open(url) def back(self): self._last_urls.insert(0, self.url) self._browser.goBack() def forward(self): try: self.visit(self._last_urls.pop()) except IndexError: pass def reload(self): self._browser.reload() def quit(self): pass @property def title(self): return self._browser.title @property def html(self): return self._browser.contents @property def url(self): return self._browser.url def find_option_by_value(self, value): html = lxml.html.fromstring(self.html) element = html.xpath('//option[@value="%s"]' % value)[0] control = self._browser.getControl(element.text) return ElementList([ZopeTestBrowserOptionElement(control, self)], find_by="value", query=value) def find_option_by_text(self, text): html = lxml.html.fromstring(self.html) element = html.xpath('//option[normalize-space(text())="%s"]' % text)[0] control = self._browser.getControl(element.text) return ElementList([ZopeTestBrowserOptionElement(control, self)], find_by="text", query=text) def find_by_css(self, selector): xpath = CSSSelector(selector).path return self.find_by_xpath(xpath, original_find="css", original_selector=selector) def find_by_xpath(self, xpath, original_find=None, original_selector=None): html = lxml.html.fromstring(self.html) elements = [] for xpath_element in html.xpath(xpath): if self._element_is_link(xpath_element): return self.find_link_by_text(xpath_element.text) elif self._element_is_control(xpath_element): return self.find_by_name(xpath_element.name) else: elements.append(xpath_element) find_by = original_find or "xpath" query = original_selector or xpath return ElementList( [ZopeTestBrowserElement(element, self) for element in elements], find_by=find_by, query=query ) def find_by_tag(self, tag): return self.find_by_xpath("//%s" % tag, original_find="tag", original_selector=tag) def find_by_value(self, value): return self.find_by_xpath('//*[@value="%s"]' % value, original_find="value", original_selector=value) def find_by_id(self, id_value): return self.find_by_xpath('//*[@id="%s"][1]' % id_value, original_find="id", original_selector=id_value) def find_by_name(self, name): elements = [] index = 0 while True: try: control = self._browser.getControl(name=name, index=index) elements.append(control) index += 1 except IndexError: break return ElementList( [ZopeTestBrowserControlElement(element, self) for element in elements], find_by="name", query=name ) def find_link_by_text(self, text): return self._find_links_by_xpath("//a[text()='%s']" % text) def find_link_by_href(self, href): return self._find_links_by_xpath("//a[@href='%s']" % href) def find_link_by_partial_href(self, partial_href): return self._find_links_by_xpath("//a[contains(@href, '%s')]" % partial_href) def find_link_by_partial_text(self, partial_text): return self._find_links_by_xpath("//a[contains(text(), '%s')]" % partial_text) def fill(self, name, value): self.find_by_name(name=name).first._control.value = value def fill_form(self, field_values): for name, value in field_values.items(): element = self.find_by_name(name) control = element.first._control if control.type == "text": control.value = value elif control.type == "checkbox": if value: control.value = control.options else: control.value = [] elif control.type == "radio": control.value = [option for option in control.options if option == value] elif control.type == "select": control.value = [value] def choose(self, name, value): control = self._browser.getControl(name=name) control.value = [option for option in control.options if option == value] def check(self, name): control = self._browser.getControl(name=name) control.value = control.options def uncheck(self, name): control = self._browser.getControl(name=name) control.value = [] def attach_file(self, name, file_path): control = self._browser.getControl(name=name) content_type, _ = mimetypes.guess_type(file_path) control.add_file(open(file_path), content_type, None) def _find_links_by_xpath(self, xpath): html = lxml.html.fromstring(self.html) links = html.xpath(xpath) return ElementList([ZopeTestBrowserLinkElement(link, self) for link in links], find_by="xpath", query=xpath) def select(self, name, value): self.find_by_name(name).first._control.value = [value] def _element_is_link(self, element): return element.tag == "a" def _element_is_control(self, element): return hasattr(element, "type") @property def cookies(self): return self._cookie_manager
class Blogger(object): def __init__(self, options): self.oron_folder_url = options.url self.title_base = options.title_base self.links_per_post = options.links_per_post self.screenshots_dir = options.screenshots self.output_dir = options.output # Email Options self.smtp_host = options.smtp_host self.smtp_port = options.smtp_port self.smtp_user = options.smtp_user self.smtp_pass = options.smtp_pass self.smtp_from = options.smtp_from self.smtp_use_tls = options.smtp_use_tls self.email_recipient = options.email self.email_initial_text = options.email_initial_text self.email_ending_text = options.email_ending_text self.start_on_post = options.start_on_post log.info("Starting Browser") self.browser = Browser("http://oron.com") self.total_links = 0 self.links = {} def login(self): if self.browser and self.browser.contents: if self.username in self.browser.contents and \ 'Logout' in self.browser.contents: log.info("Already logged in!") return log.info("Logging into Oron.com") self.browser.open('http://oron.com/login') self.browser.getControl(name='login').value = self.username self.browser.getControl(name='password').value = self.password self.browser.getForm(name='FL').submit(' Submit ') if self.username not in self.browser.contents: log.error("Failed to login...") print self.browser.contents sys.exit(1) log.info("Logged in successfully") def grab_links(self): log.info("Loading oron folder url: %s", self.oron_folder_url) self.browser.open(self.oron_folder_url) doc = etree.HTML(self.browser.contents) self.total_links = len(doc.xpath( '//table[@class="tbl2"]/tr/td[1]/a[@target="_blank"]/small' )) self.total_posts = int(round(self.total_links/(self.links_per_post*1.0))) log.info("Found %d links to create %s blog posts.", self.total_links, self.total_posts) trs = doc.xpath('//table[@class="tbl2"]/tr') for tr in trs: link_td = tr.xpath('td[1]/a') size_td = tr.xpath('td[2]/small/nobr/text()') if not link_td or not size_td: continue filename_search = link_td[0].xpath('small/text()') if not filename_search: continue filename = filename_search[0] href = link_td[0].attrib['href'] size = size_td[0].upper() self.links[filename] = { 'href': href, 'size': size, 'filename': filename } def create_posts(self): self.grab_links() filenames = sorted(self.links.keys()) chunker = Chunker(self.links_per_post) n = 1 if n < self.start_on_post: log.info("Skipping posts %d to %d", n, self.start_on_post) for chunk in chunker(filenames): if n < self.start_on_post: n += 1 continue log.info("Processing post %s of %s", n, self.total_posts) title = self.title_base + ' - %d of %d' % (n, self.total_posts) links = [] for filename in chunk: screenshot_name = filename + '.jpg' screenshot_path = os.path.join(self.screenshots_dir, screenshot_name) image_html = "<em>Missing Image preview for <b>%s</b></em>" % filename if os.path.isfile(screenshot_path): log.info("Uploading screenshot %s", screenshot_name) browser = Browser("http://www.freeporndumpster.com/legacy.php") caption_control = browser.getControl(name='imagename[]', index=0) caption_control.value = self.links[filename]['href'] file_control = browser.getControl(name='images[]', index=0) file_control.add_file( open(screenshot_path), "image/jpeg", screenshot_name ) browser.getControl('upload').click() doc = etree.HTML(browser.contents) image_html = "<em>Missing Image html for <b>%s</b></em>" % filename image_html_match = doc.xpath('//table/tr/td/p[contains(., "website")]/textarea/text()') if image_html_match: image_html = image_html_match[0] else: log.error("Failed to get uploaded image html") else: log.warn("There's no screenshot for filename %s", filename) links.append(( filename, self.links[filename]['size'], self.links[filename]['href'], image_html )) html = HTML_TEMPLATE.render( links=links, initial_text=self.email_initial_text, ending_text=self.email_ending_text ) open(os.path.join(self.output_dir, title+'.txt'), 'w').write(html) n += 1 server = None email = MIMEMultipart('alternative') email['Subject'] = title email['From'] = '*****@*****.**' email['To'] = self.email_recipient email.attach(MIMEText(html, 'html')) log.info("Sending post email") try: try: # Python 2.6 server = smtplib.SMTP(self.smtp_host, self.smtp_port, timeout=60) except: # Python 2.5 server = smtplib.SMTP(self.smtp_host, self.smtp_port) except Exception, err: log.error("There was an error sending the notification email: %s", err) if server is None: log.error("Failed to setup SMTP server") continue security_enabled = self.smtp_use_tls if security_enabled: server.ehlo() if not server.esmtp_features.has_key('starttls'): log.warning("TLS/SSL enabled but server does not support it") else: server.starttls() server.ehlo() if self.smtp_user and self.smtp_pass: try: server.login(self.smtp_user, self.smtp_pass) except smtplib.SMTPHeloError, err: log.error("The server didn't reply properly to the helo " "greeting: %s", err) except smtplib.SMTPAuthenticationError, err: log.error("The server didn't accept the username/password " "combination: %s", err)
class OronDownloader(object): def __init__(self, downloads_urls, urls_file, username, password, dest_dir=None, generate_thumbs=True, font_path=None, stop_at_quota=500): self.downloads_urls = downloads_urls self.urls_file = urls_file self.username = username self.password = password if dest_dir is None: dest_dir = os.getcwd() self.dest_dir = dest_dir self.generate_thumbs = generate_thumbs self.font_path = font_path self.stop_at_quota = stop_at_quota log.info("Starting Browser") self.browser = Browser("http://oron.com") self.to_download = 0 self.downloaded = 0 self.download_quota = 0 self.downloads_until_quota_reload = 10 def login(self): if self.browser and self.browser.contents: if self.username in self.browser.contents and \ 'Logout' in self.browser.contents: log.info("Already logged in!") return log.info("Logging into Oron.com") self.browser.open('http://oron.com/login') self.browser.getControl(name='login').value = self.username self.browser.getControl(name='password').value = self.password self.browser.getForm(name='FL').submit(' Submit ') if self.username not in self.browser.contents: log.error("Failed to login...") print self.browser.contents sys.exit(1) log.info("Logged in successfully") self.load_download_quota() def load_download_quota(self): log.info("Loading download quota...") self.browser.open("http://oron.com/?op=my_account") doc = etree.HTML(self.browser.contents) quota_string_match = doc.xpath('//form/table/tr[3]/td[2]/text()') if quota_string_match: self.download_quota = int(quota_string_match[0].split()[0]) log.info("You have %d Mb of download quota.", self.download_quota) else: log.error("Failed to retrieve download quota!") sys.exit(1) def check_quota(self): if self.download_quota <= self.stop_at_quota: log.info("Download quota is now %d. Stop downloading for now...", self.download_quota) sys.exit(0) if not self.downloads_until_quota_reload: self.downloads_until_quota_reload = 10 self.load_download_quota() else: self.downloads_until_quota_reload -= 1 def download(self): self.login() self.check_quota() if self.urls_file: log.info("Opening download URLs file: %s", self.urls_file) self.find_download_links_from_file() else: log.info("Processing %d oron folder urls", len(self.downloads_urls)) for url in self.downloads_urls: log.info("Opening downloads URL: %s", url) self.browser.open(url) self.find_download_links(self.browser.contents) def find_download_links_from_file(self): urls = open(self.urls_file, 'r').readlines(True) self.to_download = len([u for u in urls if u.strip()]) log.info("Processing %d oron file urls", self.to_download) for url in urls: if not url.strip(): continue url = url.strip() log.info('-'*78) log.info("[%d/%d] Opening URL: %s", self.downloaded+1, self.to_download, url) attempts = 0 while True: try: self.browser.open(url.strip()) doc = etree.HTML(self.browser.contents) break except socket.timeout: if attempts > 2: break log.info("Socket timeout. retrying...") attempts += 1 if attempts > 2: log.info("Too many timeouts. Continue to next...") continue info_node = node = doc.xpath('//div[@class="content"]/form/table/tr/td') #log.info("Info node match: %s", info_node) if info_node: info_node = info_node[0] else: file_not_found_match = doc.xpath('//h2/text()') if file_not_found_match: log.error("File was not found on server. Skipping...") else: log.error("Could not find out the filename. Skipping...") continue filename_match = info_node.xpath('./b/text()') #log.info("filename match is %s", filename_match) if filename_match: filename = filename_match[0].encode('utf8') size = info_node.xpath('./text()')[-1].strip().split('File size:')[-1].strip().upper() fpath = os.path.join(self.dest_dir, filename) spath = os.path.join(self.dest_dir, 'seen', filename) upath = os.path.join(self.dest_dir, 'upped', filename) uspath = os.path.join(self.dest_dir, 'upped-n-seen', filename) usnpath = os.path.join(self.dest_dir, 'upped-not-seen', filename) process_next = False for path in (fpath, spath, upath, uspath, usnpath): try: if os.path.isfile(path): if self.humanize_bytes(os.path.getsize(path)) == size: log.info("Filename %s already downloaded(%s). Skipping...", filename, path) self.downloaded += 1 process_next = True break log.info("Downloaded %s/%s of %s so far. Continuing...", self.humanize_bytes(os.path.getsize(path)), size, filename) except UnicodeEncodeError: print 'Unicode error for filename', filename process_next = True break if process_next: continue errors = 0 while errors <= 3: log.info("Processing[%s] %s from %s", errors, filename, url) try: self.download_link(url) break except (socket.error, urllib2.URLError), err: log.error("Failed to download link: %s", err) errors += 1 else: log.info("Too many errors, continue to next link") continue if self.generate_thumbs: self.generate_thumb(filename) self.download_quota -= round(float( os.path.getsize(os.path.join(self.dest_dir, filename)) )/1024/1024, 1) if self.download_quota <= self.stop_at_quota: log.info("Download quota is now %d. Stop downloading for now...", self.download_quota) sys.exit(0) if not self.downloads_until_quota_reload: self.downloads_until_quota_reload = 10 self.load_download_quota() else: self.downloads_until_quota_reload -= 1