Esempio n. 1
0
class BlackboardSession:
    def __init__(self, cookiejar, username, course_id):
        self.cookiejar_filename = cookiejar
        self.username = username
        self.course_id = course_id

        self.password = None
        self.cookies = LWPCookieJar(cookiejar)
        self.session = requests.Session()
        self.load_cookies()

    def load_cookies(self):
        try:
            self.cookies.load(ignore_discard=True)
        except FileNotFoundError:
            pass
        requests.cookies.merge_cookies(self.session.cookies, self.cookies)

    def save_cookies(self):
        requests.cookies.merge_cookies(self.cookies, self.session.cookies)
        self.cookies.save(ignore_discard=True)

    def get_cookie(self, key, path):
        try:
            return self.session.cookies._cookies['bb.au.dk'][path][key].value
        except KeyError:
            print(self.session.cookies._cookies)
            raise

    def get_username(self):
        return input("WAYF username: "******"fetch.py WAYF", self.username)
        if p is None:
            print("Please enter password for %s to store in keyring." %
                  self.username)
            p = getpass.getpass()
            keyring.set_password("fetch.py WAYF", self.username, p)
        return p

    def get_auth(self):
        if self.username is None:
            self.username = self.get_username()
        if self.password is None:
            self.password = self.get_password()
        return dict(username=self.username, password=self.password)

    def forget_password(self):
        if self.username is None:
            raise ValueError("forget_password: username is None")
        keyring.delete_password("fetch.py WAYF", self.username)

    def wayf_login(self, response):
        """Login to WAYF.

        Parameters
        ----------
        response : requests.Response
            Login page (with username/password form)
        """

        history = list(response.history) + [response]
        logger.info("Sending login details to WAYF")
        response = self.session.post(response.url, self.get_auth())
        history += list(response.history) + [response]
        if 'Forkert brugernavn eller kodeord' in response.text:
            raise BadAuth()

        response = self.post_hidden_form(response)
        history += list(response.history) + [response]
        response = self.post_hidden_form(response)
        history += list(response.history) + [response]
        response.history = history[:-1]
        return response

    def relogin(self):
        url = ('https://bb.au.dk/webapps/bb-auth-provider-shibboleth-BBLEARN' +
               '/execute/shibbolethLogin?authProviderId=_102_1')
        response = self.get(url)
        if self.detect_login(response) is False:
            logger.error("Seems logged out after re-login. " +
                         "Try deleting your cookiejar.")
            raise ParserError("Not logged in", response)
        return response

    def detect_login(self, response):
        document = html5lib.parse(response.content, encoding=response.encoding)
        logged_out_url = (
            '/webapps/portal/execute/tabs/tabAction?tab_tab_group_id=_21_1')
        o = document.find('.//h:a[@href="%s"]' % logged_out_url, NS)
        if o is not None:
            return False
        log_out_id = 'topframe.logout.label'
        o = document.find('.//h:a[@id="%s"]' % log_out_id, NS)
        if o is not None:
            return True

    def post_hidden_form(self, response):
        """Send POST request to form with only hidden fields.

        Parameters
        ----------
        response : requests.Response
            Page containing form with only hidden fields
        """

        document = html5lib.parse(response.content, encoding=response.encoding)
        form = document.find('.//h:form', NS)
        url = form.get('action')
        inputs = form.findall('.//h:input[@name]', NS)
        if not inputs:
            raise ParserError("No <input> with name", response)
        post_data = {i.get('name'): i.get('value') for i in inputs}
        response = self.session.post(url, post_data)

        return response

    def follow_html_redirect(self, response):
        """Repeatedly follow HTML redirects in the page.

        If the given response has no HTML redirect, return it unaltered.
        Otherwise, return a new response by following the redirects.
        """

        js_redirect_pattern = (r'(?:<!--)?\s*' +
                               r'document\.location\.replace\(\'' +
                               r'(?P<url>(?:\\.|[^\'])+)' + r'\'\);\s*' +
                               r'(?:(?://)?-->)?\s*$')
        real_login_url = (
            'https://bb.au.dk/webapps/' +
            'bb-auth-provider-shibboleth-BBLEARN/execute/shibbolethLogin')
        history = list(response.history) + [response]

        while True:
            document = html5lib.parse(response.content,
                                      encoding=response.encoding)
            scripts = document.findall('.//h:script', NS)

            next_url = None
            for s in scripts:
                t = ''.join(s.itertext())
                mo = re.match(js_redirect_pattern, t)
                if mo:
                    next_url = mo.group('url')
                    break
            if next_url is not None:
                o = urlparse(next_url)
                p = o.netloc + o.path
                if p == 'bb.au.dk/webapps/login/':
                    qs = parse_qs(o.query)
                    try:
                        return_url = qs['new_loc'][0]
                    except KeyError:
                        print("We are being redirected to %r" % (next_url, ))
                        return_url = ''
                    # It seems that making a GET request to this page
                    # logs you out?
                    if return_url == '/webapps/login/?action=relogin':
                        logger.debug("Not setting returnUrl to %r", return_url)
                        return_url = ''
                    new_qs = urlencode(
                        dict(returnUrl=return_url, authProviderId='_102_1'))
                    next_url = '%s?%s' % (real_login_url, new_qs)
                response = self.session.get(next_url)
                history += list(response.history) + [response]
                continue
            break
        response.history = history[:-1]
        return response

    def autologin(self, response):
        """Automatically log in if necessary.

        If the given response is not for a login form,
        just follow HTML redirects and return the response.
        Otherwise, log in using wayf_login and get_auth.
        """

        response = self.follow_html_redirect(response)
        o = urlparse(response.url)
        if o.netloc == 'wayf.au.dk':
            response = self.wayf_login(response)
        return response

    def get_edit_mode(self, response):
        document = html5lib.parse(response.content, encoding=response.encoding)
        mode_switch = document.find('.//*[@id="editModeToggleLink"]', NS)
        if mode_switch is not None:
            return 'read-on' in (mode_switch.get('class') or '').split()

    def ensure_edit_mode(self, response):
        if self.get_edit_mode(response) is False:
            url = ('https://bb.au.dk/webapps/blackboard/execute/' +
                   'doCourseMenuAction?cmd=setDesignerParticipantViewMode' +
                   '&courseId=' + self.course_id + '&mode=designer')
            logger.debug("Switch to edit mode")
            r = self.get(url)
            history = (list(response.history) + [response] + list(r.history) +
                       [r])
            response = self.get(history[0].url)
            response.history = history + list(response.history)
        return response

    def get(self, url):
        response = self.autologin(self.session.get(url))
        if self.detect_login(response) is False:
            history = response.history + [response]
            relogin_response = self.relogin()
            history += relogin_response.history + [relogin_response]
            response = self.autologin(self.session.get(url))
            response.history = history + list(response.history)
        if response.url != url:
            history = list(response.history) + [response]
            response = self.session.get(url)
            response.history = history + list(response.history)
        self.log_error(response)
        return response

    def log_error(self, response):
        document = html5lib.parse(response.content, encoding=response.encoding)
        content = document.find('.//h:div[@id="contentPanel"]', NS)
        if content is not None:
            class_list = (content.get('class') or '').split()
            if 'error' in class_list:
                logger.info("contentPanel indicates an error has occurred")
                # raise ParserError("Error", response)

    def post(self, url, data, files=None, headers=None):
        response = self.session.post(url,
                                     data=data,
                                     files=files,
                                     headers=headers)
        # if response.history:
        #     logger.warning('POST %r redirected', url)
        #     for r in response.history:
        #         logger.warning("... from %r", r.url)
        #     logger.warning("... to %r", response.url)
        return response

    def ensure_logged_in(self):
        url = (
            'https://bb.au.dk/webapps/blackboard/content/manageDashboard.jsp' +
            '?course_id=%s' % self.course_id +
            '&sortCol=LastLoginCol&sortDir=D')
        self.get(url)
Esempio n. 2
0
class BlackboardSession:
    def __init__(self, cookiejar, username, course_id):
        self.cookiejar_filename = cookiejar
        self.username = username
        self.course_id = course_id

        self.password = None
        self.cookies = LWPCookieJar(cookiejar)
        self.session = requests.Session()
        self.load_cookies()

    def load_cookies(self):
        try:
            self.cookies.load(ignore_discard=True)
        except FileNotFoundError:
            pass
        requests.cookies.merge_cookies(self.session.cookies, self.cookies)

    def save_cookies(self):
        requests.cookies.merge_cookies(self.cookies, self.session.cookies)
        self.cookies.save(ignore_discard=True)

    def get_cookie(self, key, path):
        try:
            return self.session.cookies._cookies['bb.au.dk'][path][key].value
        except KeyError:
            print(self.session.cookies._cookies)
            raise

    def get_username(self):
        return input("WAYF username: "******"fetch.py WAYF", self.username)
        if p is None:
            print("Please enter password for %s to store in keyring." %
                  self.username)
            p = getpass.getpass()
            keyring.set_password("fetch.py WAYF", self.username, p)
        return p

    def get_auth(self):
        if self.username is None:
            self.username = self.get_username()
        if self.password is None:
            self.password = self.get_password()
        return dict(username=self.username, password=self.password)

    def forget_password(self):
        if self.username is None:
            raise ValueError("forget_password: username is None")
        keyring.delete_password("fetch.py WAYF", self.username)

    def wayf_login(self, response):
        """Login to WAYF.

        Parameters
        ----------
        response : requests.Response
            Login page (with username/password form)
        """

        history = list(response.history) + [response]
        logger.info("Sending login details to WAYF")
        response = self.session.post(response.url, self.get_auth())
        history += list(response.history) + [response]
        if 'Forkert brugernavn eller kodeord' in response.text:
            raise BadAuth()

        response = self.post_hidden_form(response)
        history += list(response.history) + [response]
        response = self.post_hidden_form(response)
        history += list(response.history) + [response]
        response.history = history[:-1]
        return response

    def relogin(self):
        url = (
            'https://bb.au.dk/webapps/bb-auth-provider-shibboleth-BBLEARN' +
            '/execute/shibbolethLogin?authProviderId=_102_1')
        response = self.get(url)
        if self.detect_login(response) is False:
            logger.error("Seems logged out after re-login. " +
                         "Try deleting your cookiejar.")
            raise ParserError("Not logged in", response)
        return response

    def detect_login(self, response):
        document = html5lib.parse(response.content, encoding=response.encoding)
        logged_out_url = (
            '/webapps/portal/execute/tabs/tabAction?tab_tab_group_id=_21_1')
        o = document.find('.//h:a[@href="%s"]' % logged_out_url, NS)
        if o is not None:
            return False
        log_out_id = 'topframe.logout.label'
        o = document.find('.//h:a[@id="%s"]' % log_out_id, NS)
        if o is not None:
            return True

    def post_hidden_form(self, response):
        """Send POST request to form with only hidden fields.

        Parameters
        ----------
        response : requests.Response
            Page containing form with only hidden fields
        """

        document = html5lib.parse(response.content, encoding=response.encoding)
        form = document.find('.//h:form', NS)
        url = form.get('action')
        inputs = form.findall('.//h:input[@name]', NS)
        if not inputs:
            raise ParserError("No <input> with name", response)
        post_data = {
            i.get('name'): i.get('value')
            for i in inputs
        }
        response = self.session.post(url, post_data)

        return response

    def follow_html_redirect(self, response):
        """Repeatedly follow HTML redirects in the page.

        If the given response has no HTML redirect, return it unaltered.
        Otherwise, return a new response by following the redirects.
        """

        js_redirect_pattern = (
            r'(?:<!--)?\s*' +
            r'document\.location\.replace\(\'' +
            r'(?P<url>(?:\\.|[^\'])+)' +
            r'\'\);\s*' +
            r'(?:(?://)?-->)?\s*$')
        real_login_url = (
            'https://bb.au.dk/webapps/' +
            'bb-auth-provider-shibboleth-BBLEARN/execute/shibbolethLogin')
        history = list(response.history) + [response]

        while True:
            document = html5lib.parse(
                response.content, encoding=response.encoding)
            scripts = document.findall('.//h:script', NS)

            next_url = None
            for s in scripts:
                t = ''.join(s.itertext())
                mo = re.match(js_redirect_pattern, t)
                if mo:
                    next_url = mo.group('url')
                    break
            if next_url is not None:
                o = urlparse(next_url)
                p = o.netloc + o.path
                if p == 'bb.au.dk/webapps/login/':
                    qs = parse_qs(o.query)
                    try:
                        return_url = qs['new_loc'][0]
                    except KeyError:
                        print("We are being redirected to %r" % (next_url,))
                        return_url = ''
                    # It seems that making a GET request to this page
                    # logs you out?
                    if return_url == '/webapps/login/?action=relogin':
                        logger.debug(
                            "Not setting returnUrl to %r", return_url)
                        return_url = ''
                    new_qs = urlencode(
                        dict(returnUrl=return_url,
                             authProviderId='_102_1'))
                    next_url = '%s?%s' % (real_login_url, new_qs)
                response = self.session.get(next_url)
                history += list(response.history) + [response]
                continue
            break
        response.history = history[:-1]
        return response

    def autologin(self, response):
        """Automatically log in if necessary.

        If the given response is not for a login form,
        just follow HTML redirects and return the response.
        Otherwise, log in using wayf_login and get_auth.
        """

        response = self.follow_html_redirect(response)
        o = urlparse(response.url)
        if o.netloc == 'wayf.au.dk':
            response = self.wayf_login(response)
        return response

    def get_edit_mode(self, response):
        document = html5lib.parse(response.content, encoding=response.encoding)
        mode_switch = document.find('.//*[@id="editModeToggleLink"]', NS)
        if mode_switch is not None:
            return 'read-on' in (mode_switch.get('class') or '').split()

    def ensure_edit_mode(self, response):
        if self.get_edit_mode(response) is False:
            url = ('https://bb.au.dk/webapps/blackboard/execute/' +
                   'doCourseMenuAction?cmd=setDesignerParticipantViewMode' +
                   '&courseId=' + self.course_id +
                   '&mode=designer')
            logger.debug("Switch to edit mode")
            r = self.get(url)
            history = (list(response.history) + [response] +
                       list(r.history) + [r])
            response = self.get(history[0].url)
            response.history = history + list(response.history)
        return response

    def get(self, url):
        response = self.autologin(self.session.get(url))
        if self.detect_login(response) is False:
            history = response.history + [response]
            relogin_response = self.relogin()
            history += relogin_response.history + [relogin_response]
            response = self.autologin(self.session.get(url))
            response.history = history + list(response.history)
        if response.url != url:
            history = list(response.history) + [response]
            response = self.session.get(url)
            response.history = history + list(response.history)
        self.log_error(response)
        return response

    def log_error(self, response):
        document = html5lib.parse(response.content, encoding=response.encoding)
        content = document.find('.//h:div[@id="contentPanel"]', NS)
        if content is not None:
            class_list = (content.get('class') or '').split()
            if 'error' in class_list:
                logger.info("contentPanel indicates an error has occurred")
                # raise ParserError("Error", response)

    def post(self, url, data, files=None, headers=None):
        response = self.session.post(
            url, data=data, files=files, headers=headers)
        # if response.history:
        #     logger.warning('POST %r redirected', url)
        #     for r in response.history:
        #         logger.warning("... from %r", r.url)
        #     logger.warning("... to %r", response.url)
        return response

    def ensure_logged_in(self):
        url = (
            'https://bb.au.dk/webapps/blackboard/content/manageDashboard.jsp' +
            '?course_id=%s' % self.course_id +
            '&sortCol=LastLoginCol&sortDir=D')
        self.get(url)
Esempio n. 3
0
class StreamSession(object):
    """
    Top-level stream session interface

    Individual stream providers can be implemented by inheriting from this class
    and implementing methods for login flow, getting streams, etc.
    """

    # SESSION_FILE=os.path.join(config.CONFIG_DIR, "session")

    HEADERS = {"User-agent": USER_AGENT}

    def __init__(self, provider_id, proxies=None, *args, **kwargs):

        self.provider_id = provider_id
        self.session = requests.Session()
        self.cookies = LWPCookieJar()
        if not os.path.exists(self.COOKIES_FILE):
            self.cookies.save(self.COOKIES_FILE)
        self.cookies.load(self.COOKIES_FILE, ignore_discard=True)
        self.session.headers = self.HEADERS
        self._state = AttrDict([("proxies", proxies)])
        if proxies:
            self.proxies = proxies
        self._cache_responses = False

    @property
    def provider(self):
        return providers.get(self.provider_id)

    def login(self):
        pass

    @classmethod
    def session_type(cls):
        return cls.__name__.replace("StreamSession", "").lower()

    @classmethod
    def _COOKIES_FILE(cls):
        return os.path.join(config.CONFIG_DIR, f"{cls.session_type()}.cookies")

    @property
    def COOKIES_FILE(self):
        return self._COOKIES_FILE()

    @classmethod
    def _SESSION_FILE(cls):
        return os.path.join(config.CONFIG_DIR, f"{cls.session_type()}.session")

    @property
    def SESSION_FILE(self):
        return self._SESSION_FILE()

    @classmethod
    def new(cls, provider_id, *args, **kwargs):
        try:
            return cls.load(provider_id, **kwargs)
        except (FileNotFoundError, TypeError):
            logger.debug(f"creating new session: {args}, {kwargs}")
            return cls(provider_id, **kwargs)

    @property
    def cookies(self):
        return self.session.cookies

    @cookies.setter
    def cookies(self, value):
        self.session.cookies = value

    @classmethod
    def destroy(cls):
        if os.path.exists(cls.COOKIES_FILE):
            os.remove(cls.COOKIES_FILE)
        if os.path.exists(cls.SESSION_FILE):
            os.remove(cls.SESSION_FILE)

    @classmethod
    def load(cls, **kwargs):
        state = yaml.load(open(cls._SESSION_FILE()), Loader=AttrDictYAMLLoader)
        logger.trace(f"load: {cls.__name__}, {state}")
        return cls(provider_id, **dict(kwargs, **state))

    def save(self):
        logger.trace(f"load: {self.__class__.__name__}, {self._state}")
        with open(self.SESSION_FILE, 'w') as outfile:
            yaml.dump(self._state, outfile, default_flow_style=False)
        self.cookies.save(self.COOKIES_FILE)

    def get_cookie(self, name):
        return requests.utils.dict_from_cookiejar(self.cookies).get(name)

    def __getattr__(self, attr):
        if attr in [
                "delete", "get", "head", "options", "post", "put", "patch"
        ]:
            # return getattr(self.session, attr)
            session_method = getattr(self.session, attr)
            return functools.partial(self.request, session_method)
        # raise AttributeError(attr)

    @db_session
    def request(self, method, url, *args, **kwargs):

        response = None
        use_cache = not self.no_cache and self._cache_responses
        # print(self.proxies)
        if use_cache:
            logger.debug("getting cached response for %s" % (url))

            e = model.CacheEntry.get(url=url)

            if e:
                # (pickled_response, last_seen) = self.cursor.fetchone()

                td = datetime.now() - e.last_seen
                if td.seconds >= self._cache_responses:
                    logger.debug("cache expired for %s" % (url))
                else:
                    response = pickle.loads(e.response)
                    logger.debug("using cached response for %s" % (url))
            else:
                logger.debug("no cached response for %s" % (url))

        if not response:
            response = method(url, *args, **kwargs)
            # logger.trace(dump.dump_all(response).encode("utf-8"))

        if use_cache and not e:
            pickled_response = pickle.dumps(response)
            e = model.CacheEntry(url=url,
                                 response=pickled_response,
                                 last_seen=datetime.now())

        return response

    @property
    def headers(self):
        return []

    @property
    def proxies(self):
        return self._state.proxies

    @proxies.setter
    def proxies(self, value):
        # Override proxy environment variables if proxies are defined on session
        if value is None:
            self.session.proxies = {}
        else:
            self.session.trust_env = (len(value) == 0)
            self._state.proxies = value
            self.session.proxies.update(value)

    @contextmanager
    def cache_responses(self, duration=model.CACHE_DURATION_DEFAULT):
        self._cache_responses = duration
        try:
            yield
        finally:
            self._cache_responses = False

    def cache_responses_short(self):
        return self.cache_responses(model.CACHE_DURATION_SHORT)

    def cache_responses_medium(self):
        return self.cache_responses(model.CACHE_DURATION_MEDIUM)

    def cache_responses_long(self):
        return self.cache_responses(model.CACHE_DURATION_LONG)
Esempio n. 4
0
class StreamSession(object):
    """
    Top-level stream session interface

    Individual stream providers can be implemented by inheriting from this class
    and implementing methods for login flow, getting streams, etc.
    """

    # SESSION_FILE=os.path.join(config.CONFIG_DIR, "session")

    HEADERS = {"User-agent": USER_AGENT}

    def __init__(self,
                 username,
                 password,
                 proxies=None,
                 no_cache=False,
                 *args,
                 **kwargs):

        self.session = requests.Session()
        self.cookies = LWPCookieJar()
        if not os.path.exists(self.COOKIES_FILE):
            self.cookies.save(self.COOKIES_FILE)
        self.cookies.load(self.COOKIES_FILE, ignore_discard=True)
        self.session.headers = self.HEADERS
        self._state = AttrDict([("username", username), ("password", password),
                                ("proxies", proxies)])
        self.no_cache = no_cache
        self._cache_responses = False
        if not os.path.exists(CACHE_FILE):
            self.cache_setup(CACHE_FILE)
        self.conn = sqlite3.connect(CACHE_FILE,
                                    detect_types=sqlite3.PARSE_DECLTYPES)
        self.cursor = self.conn.cursor()
        self.cache_purge()
        # if not self.logged_in:
        self.login()
        # logger.debug("already logged in")
        # return

    @classmethod
    def session_type(cls):
        return cls.__name__.replace("StreamSession", "").lower()

    @classmethod
    def _COOKIES_FILE(cls):
        return os.path.join(config.CONFIG_DIR, f"{cls.session_type()}.cookies")

    @property
    def COOKIES_FILE(self):
        return self._COOKIES_FILE()

    @classmethod
    def _SESSION_FILE(cls):
        return os.path.join(config.CONFIG_DIR, f"{cls.session_type()}.session")

    @property
    def SESSION_FILE(self):
        return self._SESSION_FILE()

    @classmethod
    def new(cls, **kwargs):
        try:
            return cls.load(**kwargs)
        except FileNotFoundError:
            logger.trace(f"creating new session: {kwargs}")
            provider = config.settings.profile.providers.get(
                cls.session_type())
            return cls(username=provider.username,
                       password=provider.password,
                       **kwargs)

    @property
    def cookies(self):
        return self.session.cookies

    @cookies.setter
    def cookies(self, value):
        self.session.cookies = value

    @classmethod
    def destroy(cls):
        if os.path.exists(cls.COOKIES_FILE):
            os.remove(cls.COOKIES_FILE)
        if os.path.exists(cls.SESSION_FILE):
            os.remove(cls.SESSION_FILE)

    @classmethod
    def load(cls, *args, **kwargs):
        state = yaml.load(open(cls._SESSION_FILE()), Loader=AttrDictYAMLLoader)
        logger.trace(f"load: {cls.__name__}, {state}")
        return cls(**state)

    def save(self):
        logger.trace(f"load: {self.__class__.__name__}, {self._state}")
        with open(self.SESSION_FILE, 'w') as outfile:
            yaml.dump(self._state, outfile, default_flow_style=False)
        self.cookies.save(self.COOKIES_FILE)

    def get_cookie(self, name):
        return requests.utils.dict_from_cookiejar(self.cookies).get(name)

    def __getattr__(self, attr):
        if attr in [
                "delete", "get", "head", "options", "post", "put", "patch"
        ]:
            # return getattr(self.session, attr)
            session_method = getattr(self.session, attr)
            return functools.partial(self.request, session_method)
        raise AttributeError(attr)

    def request(self, method, url, *args, **kwargs):

        response = None
        use_cache = not self.no_cache and self._cache_responses
        if use_cache:
            logger.debug("getting cached response fsesor %s" % (url))
            self.cursor.execute(
                "SELECT response, last_seen "
                "FROM response_cache "
                "WHERE url = ?", (url, ))
            try:
                (pickled_response, last_seen) = self.cursor.fetchone()
                td = datetime.now() - last_seen
                if td.seconds >= self._cache_responses:
                    logger.debug("cache expired for %s" % (url))
                else:
                    response = pickle.loads(pickled_response)
                    logger.debug("using cached response for %s" % (url))
            except TypeError:
                logger.debug("no cached response for %s" % (url))

        # if not response:
        #     response = method(url, *args, **kwargs)
        #     logger.trace(dump.dump_all(response).decode("utf-8"))
        if use_cache:
            pickled_response = pickle.dumps(response)
            sql = """INSERT OR REPLACE
            INTO response_cache (url, response, last_seen)
            VALUES (?, ?, ?)"""
            self.cursor.execute(sql, (url, pickled_response, datetime.now()))
            self.conn.commit()

        return response

    @property
    def username(self):
        return self._state.username

    @property
    def password(self):
        return self._state.password

    @property
    def proxies(self):
        return self._state.proxies

    @property
    def headers(self):
        return []

    @proxies.setter
    def proxies(self, value):
        # Override proxy environment variables if proxies are defined on session
        if value is not None:
            self.session.trust_env = (len(value) == 0)
        self._state.proxies = value
        self.session.proxies.update(value)

    @contextmanager
    def cache_responses(self, duration=CACHE_DURATION_DEFAULT):
        self._cache_responses = duration
        try:
            yield
        finally:
            self._cache_responses = False

    def cache_responses_short(self):
        return self.cache_responses(CACHE_DURATION_SHORT)

    def cache_responses_medium(self):
        return self.cache_responses(CACHE_DURATION_MEDIUM)

    def cache_responses_long(self):
        return self.cache_responses(CACHE_DURATION_LONG)

    def cache_setup(self, dbfile):

        conn = sqlite3.connect(dbfile)
        c = conn.cursor()
        c.execute('''
        CREATE TABLE response_cache
        (url TEXT,
        response TEXT,
        last_seen TIMESTAMP DEFAULT (datetime('now','localtime')),
        PRIMARY KEY (url))''')
        conn.commit()
        c.close()

    def cache_purge(self, days=CACHE_DURATION_LONG):

        self.cursor.execute("DELETE "
                            "FROM response_cache "
                            "WHERE last_seen < datetime('now', '-%d days')" %
                            (days))