def do_request(self, req, status, expect_errors): """ Override webtest.TestApp's method so that we do real HTTP requests instead of WSGI calls. """ headers = {} if self.cookies: c = BaseCookie() for name, value in self.cookies.items(): c[name] = value hc = '; '.join(['='.join([m.key, m.value]) for m in c.values()]) req.headers['Cookie'] = hc res = self._do_httplib_request(req) # Set these attributes for consistency with webtest. res.request = req res.test_app = self if not expect_errors: self._check_status(res.status_int, res) self._check_errors(res) res.cookies_set = {} for header in res.headers.getall('set-cookie'): try: c = BaseCookie(header) except CookieError, e: raise CookieError("Could not parse cookie header %r: %s" % (header, e)) for key, morsel in c.items(): self.cookies[key] = morsel.value res.cookies_set[key] = morsel.value
def do_request(self, req, status, expect_errors): """ Override webtest.TestApp's method so that we do real HTTP requests instead of WSGI calls. """ headers = {} if self.cookies: c = BaseCookie() for name, value in self.cookies.items(): c[name] = value hc = '; '.join(['='.join([m.key, m.value]) for m in c.values()]) req.headers['Cookie'] = hc res = self._do_httplib_request(req) # Set these attributes for consistency with webtest. res.request = req res.test_app = self if not expect_errors: self._check_status(res.status_int, res) self._check_errors(res) res.cookies_set = {} for header in res.headers.getall('set-cookie'): try: c = BaseCookie(header) except CookieError, e: raise CookieError( "Could not parse cookie header %r: %s" % (header, e)) for key, morsel in c.items(): self.cookies[key] = morsel.value res.cookies_set[key] = morsel.value
def do_request(self, req): errors = StringIO() req.environ['wsgi.errors'] = errors if self.cookies: cookie_header = ''.join([ '%s="%s"; ' % (name, cookie_quote(value)) for name, value in self.cookies.items()]) req.environ['HTTP_COOKIE'] = cookie_header res = req.get_response(self.application, catch_exc_info=True) # We do this to make sure the app_iter is exausted: res.body res.errors = errors.getvalue() res.cookies_set = {} for header in res.headers.getall('set-cookie'): try: c = BaseCookie(header) except CookieError, e: raise CookieError( "Could not parse cookie header %r: %s" % (header, e)) for key, morsel in c.items(): self.cookies[key] = morsel.value res.cookies_set[key] = morsel.value
def do_request(self, req, status, expect_errors): """ Executes the given request (``req``), with the expected ``status``. Generally ``.get()`` and ``.post()`` are used instead. """ __tracebackhide__ = True errors = StringIO() req.environ['wsgi.errors'] = errors if self.cookies: c = BaseCookie() for name, value in self.cookies.items(): c[name] = value req.environ['HTTP_COOKIE'] = str(c).split(': ', 1)[1] req.environ['paste.testing'] = True req.environ['paste.testing_variables'] = {} app = lint.middleware(self.app) old_stdout = sys.stdout out = CaptureStdout(old_stdout) try: sys.stdout = out start_time = time.time() ## FIXME: should it be an option to not catch exc_info? res = req.get_response(app, catch_exc_info=True) end_time = time.time() finally: sys.stdout = old_stdout sys.stderr.write(out.getvalue()) res.app = app res.test_app = self # We do this to make sure the app_iter is exausted: res.body res.errors = errors.getvalue() total_time = end_time - start_time for name, value in req.environ['paste.testing_variables'].items(): if hasattr(res, name): raise ValueError( "paste.testing_variables contains the variable %r, but " "the response object already has an attribute by that " "name" % name) setattr(res, name, value) if not expect_errors: self._check_status(status, res) self._check_errors(res) res.cookies_set = {} for header in res.headers.getall('set-cookie'): try: c = BaseCookie(header) except CookieError, e: raise CookieError("Could not parse cookie header %r: %s" % (header, e)) for key, morsel in c.items(): self.cookies[key] = morsel.value res.cookies_set[key] = morsel.value
def do_request(self, req, status, expect_errors): """ Executes the given request (``req``), with the expected ``status``. Generally ``.get()`` and ``.post()`` are used instead. """ __tracebackhide__ = True errors = StringIO() req.environ["wsgi.errors"] = errors if self.cookies: c = BaseCookie() for name, value in self.cookies.items(): c[name] = value req.environ["HTTP_COOKIE"] = str(c).split(": ", 1)[1] req.environ["paste.testing"] = True req.environ["paste.testing_variables"] = {} app = lint.middleware(self.app) old_stdout = sys.stdout out = CaptureStdout(old_stdout) try: sys.stdout = out start_time = time.time() ## FIXME: should it be an option to not catch exc_info? res = req.get_response(app, catch_exc_info=True) end_time = time.time() finally: sys.stdout = old_stdout sys.stderr.write(out.getvalue()) res.app = app res.test_app = self # We do this to make sure the app_iter is exausted: res.body res.errors = errors.getvalue() total_time = end_time - start_time for name, value in req.environ["paste.testing_variables"].items(): if hasattr(res, name): raise ValueError( "paste.testing_variables contains the variable %r, but " "the response object already has an attribute by that " "name" % name ) setattr(res, name, value) if not expect_errors: self._check_status(status, res) self._check_errors(res) res.cookies_set = {} for header in res.headers.getall("set-cookie"): try: c = BaseCookie(header) except CookieError, e: raise CookieError("Could not parse cookie header %r: %s" % (header, e)) for key, morsel in c.items(): self.cookies[key] = morsel.value res.cookies_set[key] = morsel.value
def transferCookiesToSafari(): """ Copy all crunchyroll cookies from Plex's cookie storage into Safari's Plist """ import platform if "darwin" in platform.system().lower(): cookieString = HTTP.GetCookiesForURL(BASE_URL) if not cookieString: return True try: theCookies = BaseCookie(cookieString) appendThis = [] tomorrow = datetime.now() + timedelta((1)) for k, v in theCookies.items(): #Plex doesn't supply these, so: cookieDict = { 'Domain': ".crunchyroll.com", 'Path': "/", 'Expires': tomorrow, 'Created': time.time(), 'Name': k, 'Value': v.value } appendThis.append(cookieDict) #Log.Debug("#######Transferring these cookies:") #Log.Debug(appendThis) filename = os.path.expanduser("~/Library/Cookies/Cookies.plist") theList = plistlib.readPlist(filename) finalCookies = appendThis # brute force replace for item in theList: if not "crunchyroll.com" in item['Domain']: finalCookies.append(item) plistlib.writePlist(finalCookies, filename) return True except Exception, arg: Log.Error("#########transferCookiesToSafari() Exception occured:") Log.Error(repr(Exception) + " " + repr(arg)) return False
def transferCookiesToSafari(): """ Copy all crunchyroll cookies from Plex's cookie storage into Safari's Plist """ import platform if "darwin" in platform.system().lower(): cookieString = HTTP.GetCookiesForURL(BASE_URL) if not cookieString: return True try: theCookies = BaseCookie(cookieString) appendThis = [] tomorrow = datetime.now() + timedelta((1)) for k, v in theCookies.items(): #Plex doesn't supply these, so: cookieDict = {'Domain':".crunchyroll.com", 'Path':"/", 'Expires': tomorrow, 'Created': time.time(), 'Name': k, 'Value': v.value } appendThis.append(cookieDict) #Log.Debug("#######Transferring these cookies:") #Log.Debug(appendThis) filename = os.path.expanduser("~/Library/Cookies/Cookies.plist") theList = plistlib.readPlist(filename) finalCookies = appendThis # brute force replace for item in theList: if not "crunchyroll.com" in item['Domain']: finalCookies.append(item) plistlib.writePlist(finalCookies, filename) return True except Exception, arg: Log.Error("#########transferCookiesToSafari() Exception occured:") Log.Error(repr(Exception) + " " + repr(arg)) return False
class Response(object): __slots__ = [ '_content', '_content_length', '_cookies', '_headers', '_status_code' ] def __init__(self): self._content = 'None' self._content_length = None self._cookies = BaseCookie() self._headers = {HttpResponseHeaders.CONTENT_TYPE: 'text/html'} self._status_code = HttpStatusCodes.HTTP_200 def set_status(self, status_code): """Set status code for the response. Args: status_code (:obj:`str`): HTTP status See Also: :class:`drongo.status_codes.HttpStatusCodes` """ self._status_code = status_code def set_header(self, key, value): """Set a response header. Args: key (:obj:`str`): Header name value (:obj:`str`): Header value See Also: :class:`drongo.response_headers.HttpResponseHeaders` """ self._headers[key] = value def set_cookie(self, key, value, domain=None, path='/', secure=False, httponly=True): """Set a cookie. Args: key (:obj:`str`): Cookie name value (:obj:`str`): Cookie value domain (:obj:`str`): Cookie domain path (:obj:`str`): Cookie value secure (:obj:`bool`): True if secure, False otherwise httponly (:obj:`bool`): True if it's a HTTP only cookie, False otherwise """ self._cookies[key] = value if domain: self._cookies[key]['domain'] = domain if path: self._cookies[key]['path'] = path if secure: self._cookies[key]['secure'] = secure if httponly: self._cookies[key]['httponly'] = httponly def set_content(self, content, content_length=None): """Set content for the response. Args: content (:obj:`str` or :obj:`iterable`): Response content. Can be either unicode or raw bytes. When returning large content, an iterable (or a generator) can be used to avoid loading entire content into the memory. content_length (:obj:`int`, optional): Content length. Length will be determined if not set. If content is an iterable, it's a good practise to set the content length. """ if content_length is not None: self._content_length = content_length self._content = content def bake(self, start_response): """Bakes the response and returns the content. Args: start_response (:obj:`callable`): Callback method that accepts status code and a list of tuples (pairs) containing headers' key and value respectively. """ if isinstance(self._content, six.text_type): self._content = self._content.encode('utf8') if self._content_length is None: self._content_length = len(self._content) self._headers[HttpResponseHeaders.CONTENT_LENGTH] = \ str(self._content_length) headers = list(self._headers.items()) cookies = [(HttpResponseHeaders.SET_COOKIE, v.OutputString()) for _, v in self._cookies.items()] if len(cookies): headers = list(headers) + cookies start_response(self._status_code, headers) if isinstance(self._content, six.binary_type): return [self._content] return self._content # Helper functions def set_redirect(self, url, status=HttpStatusCodes.HTTP_303): """Helper method to set a redirect response. Args: url (:obj:`str`): URL to redirect to status (:obj:`str`, optional): Status code of the response """ self.set_status(status) self.set_content('') self.set_header(HttpResponseHeaders.LOCATION, url) def set_json(self, obj, status=HttpStatusCodes.HTTP_200): """Helper method to set a JSON response. Args: obj (:obj:`object`): JSON serializable object status (:obj:`str`, optional): Status code of the response """ obj = json.dumps(obj, sort_keys=True, default=lambda x: str(x)) self.set_status(status) self.set_header(HttpResponseHeaders.CONTENT_TYPE, 'application/json') self.set_content(obj)
def do_request(self, req, status, expect_errors): """ Executes the given request (``req``), with the expected ``status``. Generally ``.get()`` and ``.post()`` are used instead. To use this:: resp = app.do_request(webtest.TestRequest.blank( 'url', ...args...)) Note you can pass any keyword arguments to ``TestRequest.blank()``, which will be set on the request. These can be arguments like ``content_type``, ``accept``, etc. """ __tracebackhide__ = True errors = StringIO() req.environ['wsgi.errors'] = errors if self.cookies: cookie_header = ''.join([ '%s="%s"; ' % (name, cookie_quote(value)) for name, value in self.cookies.items()]) req.environ['HTTP_COOKIE'] = cookie_header req.environ['paste.testing'] = True req.environ['paste.testing_variables'] = {} app = lint.middleware(self.app) old_stdout = sys.stdout out = CaptureStdout(old_stdout) try: sys.stdout = out start_time = time.time() ## FIXME: should it be an option to not catch exc_info? res = req.get_response(app, catch_exc_info=True) end_time = time.time() finally: sys.stdout = old_stdout res.app = app res.test_app = self # We do this to make sure the app_iter is exausted: res.body res.errors = errors.getvalue() total_time = end_time - start_time for name, value in req.environ['paste.testing_variables'].items(): if hasattr(res, name): raise ValueError( "paste.testing_variables contains the variable %r, but " "the response object already has an attribute by that " "name" % name) setattr(res, name, value) if not expect_errors: self._check_status(status, res) self._check_errors(res) res.cookies_set = {} for header in res.headers.getall('set-cookie'): try: c = BaseCookie(header) except CookieError, e: raise CookieError( "Could not parse cookie header %r: %s" % (header, e)) for key, morsel in c.items(): self.cookies[key] = morsel.value res.cookies_set[key] = morsel.value
class TestAgent(object): """ A ``TestAgent`` object provides a user agent for the WSGI application under test. Key methods and properties: - ``get(path)``, ``post(path)``, ``post_multipart`` - create get/post requests for the WSGI application and return a new ``TestAgent`` object - ``request``, ``response`` - the `werkzeug` request and response objects associated with the last WSGI request. - ``body`` - the body response as a string - ``lxml`` - the lxml representation of the response body (only applicable for HTML responses) - ``reset()`` - reset the TestAgent object to its initial state, discarding any form field values - ``find()`` (or dictionary-style attribute access) - evalute the given xpath expression against the current response body and return a list. """ response_class = wz.Response _lxml= None environ_defaults = { 'SCRIPT_NAME': "", 'PATH_INFO': "", 'QUERY_STRING': "", 'SERVER_NAME': "localhost", 'SERVER_PORT': "80", 'SERVER_PROTOCOL': "HTTP/1.0", 'REMOTE_ADDR': '127.0.0.1', 'wsgi.version': (1, 0), 'wsgi.url_scheme': 'http', 'wsgi.multithread': False, 'wsgi.multiprocess': False, 'wsgi.run_once': False, } def __init__(self, app, request=None, response=None, cookies=None, history=None, validate_wsgi=False): # TODO: Make validate_wsgi pass if validate_wsgi: app = wsgi_validator(app) self.app = app self.request = request self.response = response self._elements = [] # Stores file upload field values in forms self.file_uploads = {} if cookies: self.cookies = cookies else: self.cookies = BaseCookie() if response: self.cookies.update(parse_cookies(response)) if history: self.history = history else: self.history = [] @classmethod def make_environ(cls, REQUEST_METHOD='GET', PATH_INFO='', wsgi_input='', **kwargs): SCRIPT_NAME = kwargs.pop('SCRIPT_NAME', cls.environ_defaults["SCRIPT_NAME"]) if SCRIPT_NAME and SCRIPT_NAME[-1] == "/": SCRIPT_NAME = SCRIPT_NAME[:-1] PATH_INFO = "/" + PATH_INFO if not SCRIPT_NAME: assert not PATH_INFO.startswith('.') environ = cls.environ_defaults.copy() environ.update(kwargs) for key, value in kwargs.items(): environ[key.replace('wsgi_', 'wsgi.')] = value if isinstance(wsgi_input, basestring): wsgi_input = StringIO(wsgi_input) environ.update({ 'REQUEST_METHOD': REQUEST_METHOD, 'SCRIPT_NAME': SCRIPT_NAME, 'PATH_INFO': PATH_INFO, 'wsgi.input': wsgi_input, 'wsgi.errors': StringIO(), }) if environ['SCRIPT_NAME'] == '/': environ['SCRIPT_NAME'] = '' environ['PATH_INFO'] = '/' + environ['PATH_INFO'] while PATH_INFO.startswith('//'): PATH_INFO = PATH_INFO[1:] return environ def _request(self, environ, follow=False, history=False, status=None): path = environ['SCRIPT_NAME'] + environ['PATH_INFO'] environ['HTTP_COOKIE'] = '; '.join( '%s=%s' % (key, morsel.value) for key, morsel in self.cookies.items() if path.startswith(morsel['path']) ) if '?' in environ['PATH_INFO']: environ['PATH_INFO'], querystring = environ['PATH_INFO'].split('?', 1) if environ.get('QUERY_STRING'): environ['QUERY_STRING'] += querystring else: environ['QUERY_STRING'] = querystring if history: history = self.history + [self] else: history = self.history response = self.response_class.from_app(self.app, environ) agent = self.__class__(self.app, wz.Request(environ), response, self.cookies, history, validate_wsgi=False) if status and (status != response.status): raise BadResponse(response.status, status) if response.status == "404 NOT FOUND": if not status == response.status: raise PageNotFound(path) if follow: return agent.follow_all() return agent def get(self, PATH_INFO='/', data=None, charset='UTF-8', follow=False, history=True, status=None, **kwargs): """ Make a GET request to the application and return the response. """ if data is not None: kwargs.setdefault('QUERY_STRING', wz.url_encode(data, charset=charset, separator='&')) if self.request: PATH_INFO = uri_join_same_server(self.request.url, PATH_INFO) return self._request( self.make_environ('GET', PATH_INFO=PATH_INFO, **kwargs), follow, history, status=status, ) def post(self, PATH_INFO='/', data=None, charset='UTF-8', follow=False, history=True, status=None, **kwargs): """ Make a POST request to the application and return the response. """ if data is None: data = [] if self.request: PATH_INFO = uri_join_same_server(self.request.url, PATH_INFO) data = wz.url_encode(data, charset=charset, separator='&') wsgi_input = StringIO(data) wsgi_input.seek(0) return self._request( self.make_environ( 'POST', PATH_INFO=PATH_INFO, CONTENT_TYPE="application/x-www-form-urlencoded", CONTENT_LENGTH=str(len(data)), wsgi_input=wsgi_input, **kwargs ), follow, history, status=status, ) def post_multipart(self, PATH_INFO='/', data=None, files=None, charset='UTF-8', follow=False, **kwargs): """ Create a MockWSGI configured to post multipart/form-data to the given URI. This is usually used for mocking file uploads data dictionary of post data files list of ``(name, filename, content_type, data)`` tuples. ``data`` may be either a byte string, iterator or file-like object. """ if data is None: data = {} if files is None: files = [] if self.request: PATH_INFO = uri_join_same_server(self.request.url, PATH_INFO) boundary = '----------------------------------------BoUnDaRyVaLuE' def add_headers(key, value): """ Return a tuple of ``([(header-name, header-value), ...], data)`` for the given key/value pair """ if isinstance(value, tuple): filename, content_type, data = value headers = [ ('Content-Disposition', 'form-data; name="%s"; filename="%s"' % (key, filename)), ('Content-Type', content_type) ] return headers, data else: if isinstance(value, unicode): value = value.encode(charset) headers = [ ('Content-Disposition', 'form-data; name="%s"' % (key,)) ] return headers, value items = itertools.chain( (add_headers(k, v) for k, v in data), (add_headers(k, (fname, ctype, data)) for k, fname, ctype, data in files), ) CRLF = '\r\n' post_data = StringIO() post_data.write('--' + boundary) for headers, data in items: post_data.write(CRLF) for name, value in headers: post_data.write('%s: %s%s' % (name, value, CRLF)) post_data.write(CRLF) if hasattr(data, 'read'): copyfileobj(data, post_data) elif isinstance(data, str): post_data.write(data) else: for chunk in data: post_data.write(chunk) post_data.write(CRLF) post_data.write('--' + boundary) post_data.write('--' + CRLF) length = post_data.tell() post_data.seek(0) kwargs.setdefault('CONTENT_LENGTH', str(length)) return self._request( self.make_environ( 'POST', PATH_INFO, CONTENT_TYPE='multipart/form-data; boundary=%s' % boundary, wsgi_input=post_data, **kwargs ), follow=follow, ) def start_response(self, status, headers, exc_info=None): """ No-op implementation. """ def __str__(self): if self.response: return str(self.response) else: return super(TestAgent, self).__str__() @property def status(self): return self.response.status @property def body(self): return self.response.data @property def lxml(self): if self._lxml is not None: return self._lxml self.reset() return self._lxml def html(self, encoding=unicode): """ Return a HTML representation of the element. Defaults to returning unicode """ return lxml.html.tostring(self.lxml, encoding=encoding) def pretty(self, encoding=unicode): """ Return an pretty-printed unicode representation of the element """ return lxml.html.tostring(self.lxml, pretty_print=True, encoding=encoding) @property def root_element(self): return ElementWrapper(self, self.lxml) def reset(self): """ Reset the lxml document, abandoning any changes made """ if not self.response: raise NoRequestMadeError for element in self._elements: element.reset() self._lxml = browserify( lxml.html.fromstring( self.response.data.decode('utf-8'))) def _find(self, path, namespaces=None, css=False, **kwargs): """ Return elements matching the given xpath expression. For convenience that the EXSLT regular expression namespace (``http://exslt.org/regular-expressions``) is prebound to the prefix ``re``. """ if css: selector = CSSSelector(path) return selector(self.lxml) ns = {'re': REGEXP_NAMESPACE} if namespaces is not None: ns.update(namespaces) namespaces = ns result = self.lxml.xpath(path, namespaces=namespaces, **kwargs) return result def one(self, path, css=False, **kwargs): """ Returns the first result from Agent.all. Raises an error if more than one result is found. """ elements = self.all(path, css=css, **kwargs) if len(elements) > 1: raise MultipleMatchesError(path.encode('utf8'), elements, kwargs) elif len(elements) == 0: raise NoMatchesError(path.encode('utf8'), kwargs) else: return elements[0] def all(self, path, css=False, **kwargs): """ Returns the results of Agent.find, or Agent._findcss if css is True """ elements = self._find(path, css=css, **kwargs) return [ElementWrapper(self, el) for el in elements] @property def form(self): """ Returns a form if there is only one on the page. It is an error otherwise """ return self.one(u'//form') def click(self, path=None, follow=False, many=False, **kwargs): if not path: path = _path_from_kwargs('a', **kwargs) if many: return self.all(path)[0].click(follow=follow) else: return self.one(path).click(follow=follow) def _click(self, element, follow=False): href = element.attrib['href'] if '#' in href: href = href.split('#')[0] return self.get(href, follow=follow) def follow(self): """ If response has a ``30x`` status code, fetch (``GET``) the redirect target. No entry is recorded in the agent's history list. """ if not (300 <= int(self.response.status.split()[0]) < 400): raise AssertionError( "Can't follow non-redirect response (got %s for %s %s)" % ( self.response.status, self.request.method, self.request.path, ) ) return self.get( self.response.headers.get('Location'), history=False, ) def follow_all(self): """ If response has a ``30x`` status code, fetch (``GET``) the redirect target, until a non-redirect code is received. No entries are recorded in the agent's history list. """ agent = self while True: try: agent = agent.follow() except AssertionError: return agent def back(self, count=1): return self.history[-abs(count)] def __enter__(self): """ Provde support for context blocks """ return self def __exit__(self, exc_type, exc_value, traceback): """ At end of context block, reset the lxml document """ self.reset()