Ejemplo n.º 1
0
def jira_paginated_get(url, obj_name, session=None, start=0, retries=3, **fields):
    """
    Like ``paginated_get``, but uses JIRA's conventions for a paginated API, which
    are different from Github's conventions.
    """
    session = session or requests.Session()
    url = URLObject(url)
    more_results = True
    while more_results:
        result_url = (
            url.set_query_param("startAt", str(start))
               .set_query_params(**fields)
        )
        for _ in xrange(retries):
            try:
                result_resp = session.get(result_url)
                result = result_resp.json()
                break
            except ValueError:
                continue
        if not result_resp.ok:
            raise requests.exceptions.RequestException(result)
        result = result_resp.json()
        for obj in result[obj_name]:
            yield obj
        returned = len(result[obj_name])
        total = result["total"]
        if start + returned < total:
            start += returned
        else:
            more_results = False
Ejemplo n.º 2
0
def proxied(url):
    url = URLObject(url)
    netloc = url.netloc or settings.SERVER_NAME
    cache = get_cache()
    if netloc not in cache:
        return url
    return url.with_netloc(cache[netloc])
Ejemplo n.º 3
0
def process_url(url, tracking_id):
    amazon_url = URLObject(url)

    amazon_url = amazon_url.with_scheme('https')
    amazon_url = amazon_url.set_query_param('tag', tracking_id)

    return amazon_url
Ejemplo n.º 4
0
    def process_request(self, request):
        token = request.GET.get(self.param_name)
        if not token:
            return

        redirect_url = URLObject(request.get_full_path())
        redirect_url = redirect_url.del_query_param(self.param_name)

        response = redirect(unicode(redirect_url))
        try:
            token_data = tempus_loads(token, max_age=self.max_age)
            tempus = getattr(request, 'tempus', None)
            if tempus:
                current_tempus = tempus.copy()
                current_tempus.update(token_data)
                request.tempus = current_tempus
            else:
                request.tempus = token_data
        except SignatureExpired:
            value = self.__process_func(request, 'expired_func')
            if value:
                return value
        except BadSignature:
            value = self.__process_func(request, 'unsuccess_func')
            if value:
                return value
        else:
            value = self.__process_func(request, 'success_func')
            if value:
                return value

        add_never_cache_headers(response)
        return response
Ejemplo n.º 5
0
 def test_multiple_parses_are_idempotent(self):
     url = u'http://xn-hllo-bpa.com/path%20withspaces?query=es%25capes&foo=bar#frag%28withescapes%29'
     parse1 = URLObject.parse(url)
     self.assertEqual(unicode(url), unicode(parse1))
     parse2 = URLObject.parse(unicode(parse1))
     self.assertEqual(unicode(url), unicode(parse2))
     self.assertEqual(unicode(parse1), unicode(parse2))
Ejemplo n.º 6
0
def jira_group_members(groupname, session=None, start=0, retries=3, debug=False):
    """
    JIRA's group members API is horrible. This makes it easier to use.
    """
    session = session or requests.Session()
    url = URLObject("/rest/api/2/group").set_query_param("groupname", groupname)
    more_results = True
    while more_results:
        end = start + 49  # max 50 users per page
        expand = "users[{start}:{end}]".format(start=start, end=end)
        result_url = url.set_query_param("expand", expand)
        for _ in xrange(retries):
            try:
                if debug:
                    print(result_url, file=sys.stderr)
                result_resp = session.get(result_url)
                result = result_resp.json()
                break
            except ValueError:
                continue
        result_resp.raise_for_status()
        result = result_resp.json()
        if not result:
            break
        users = result["users"]["items"]
        for user in users:
            yield user
        returned = len(users)
        total = result["users"]["size"]
        if start + returned < total:
            start += returned
        else:
            more_results = False
Ejemplo n.º 7
0
def get_pulls(labels=None, state="open", since=None, org=False):
    url = URLObject("https://api.github.com/repos/edx/edx-platform/issues")
    if labels:
        url = url.set_query_param('labels', ",".join(labels))
    if since:
        url = url.set_query_param('since', since.isoformat())
    if state:
        url = url.set_query_param('state', state)
    url = url.set_query_param('sort', 'updated')

    org_fn = None
    if org:
        try:
            with open("mapping.yaml") as fmapping:
                user_mapping = yaml.load(fmapping)
            def_org = "other"
        except IOError:
            user_mapping = {}
            def_org = "---"

        def org_fn(issue):
            return user_mapping.get(issue["user.login"], {}).get("institution", def_org)

    issues = JPullRequest.from_json(paginated_get(url), org_fn)
    if org:
        issues = sorted(issues, key=operator.itemgetter("org"))

    return issues
Ejemplo n.º 8
0
def show_pulls(jrep, labels=None, show_comments=False, state="open", since=None, org=False):
    issues = get_pulls(labels, state, since, org)

    category = None
    for index, issue in enumerate(issues):
        issue.finish_loading()
        if issue.get("org") != category:
            # new category! print category header
            category = issue["org"]
            print("-- {category} ----".format(category=category))

        if 0:
            import pprint
            pprint.pprint(issue.obj)
        print(issue.format(ISSUE_FMT))

        if show_comments:
            comments_url = URLObject(issue['comments_url'])
            comments_url = comments_url.set_query_param("sort", "created")
            comments_url = comments_url.set_query_param("direction", "desc")
            comments = paginated_get(comments_url)
            last_five_comments = reversed(more_itertools.take(5, comments))
            for comment in last_five_comments:
                print(comment.format(COMMENT_FMT))

    # index is now set to the total number of pull requests
    print()
    print("{num} pull requests".format(num=index+1))
Ejemplo n.º 9
0
def get_pulls(owner_repo, labels=None, state="open", since=None, org=False):
    url = URLObject("https://api.github.com/repos/{}/issues".format(owner_repo))
    if labels:
        url = url.set_query_param('labels', ",".join(labels))
    if since:
        url = url.set_query_param('since', since.isoformat())
    if state:
        url = url.set_query_param('state', state)
    url = url.set_query_param('sort', 'updated')

    org_fn = None
    if org:
        try:
            with open("people.yaml") as fpeople:
                people = yaml.load(fpeople)
            def_org = "other"
        except IOError:
            people = {}
            def_org = "---"

        def org_fn(issue):
            user_info = people.get(issue["user.login"])
            if not user_info:
                user_info = {"institution": "unsigned"}
            return user_info.get("institution", def_org)

    issues = JPullRequest.from_json(paginated_get(url), org_fn)
    if org:
        issues = sorted(issues, key=operator.itemgetter("org"))

    return issues
Ejemplo n.º 10
0
    def as_value(self, data, context):
        # The queries of the current URL, not using sequences here
        # since the order of sorting arguments matter
        url = URLObject(context['request'].get_full_path())
        queries = url.query.dict

        name, orderings = data['with'], data['by']
        query = self.find_query(queries.get(name), orderings, orderings[0])
        url = url.set_query_param(name, query)

        # If this isn't a block tag we probably only want the URL
        if not self._meta.block:
            return url

        label = self.nodelist.render(context)
        if not label.strip():
            raise TemplateSyntaxError("No label was specified")

        parts = []
        for part in query.split(','):
            part = part.strip()
            if part.startswith('-'):
                part = part.lstrip('-')
                # Translators: Used in title of descending sort fields
                text = _("'%(sort_field)s' (desc)")
            else:
                # Translators: Used in title of ascending sort fields
                text = _("'%(sort_field)s' (asc)")
            parts.append(text % {'sort_field': part})
        # Translators: Used for the link/form input title excluding the sort fields
        title = (_('Sort by: %(sort_fields)s') %
                 {'sort_fields': get_text_list(parts, _('and'))})

        extra_context = dict(data, title=title, label=label, url=url, query=query)
        return render_to_string(self.using(data), extra_context, context)
Ejemplo n.º 11
0
def should_transition(issue):
    """
    Return a boolean indicating if the given issue should be transitioned
    automatically from "Needs Triage" to an open status.
    """
    issue_key = to_unicode(issue["key"])
    issue_status = to_unicode(issue["fields"]["status"]["name"])
    project_key = to_unicode(issue["fields"]["project"]["key"])
    if issue_status != "Needs Triage":
        print(
            "{key} has status {status}, does not need to be processed".format(
                key=issue_key, status=issue_status,
            ),
            file=sys.stderr,
        )
        return False

    # Open source pull requests do not skip Needs Triage.
    # However, if someone creates a subtask on an OSPR issue, that subtasks
    # might skip Needs Triage (it just follows the rest of the logic in this
    # function.)
    is_subtask = issue["fields"]["issuetype"]["subtask"]
    if project_key == "OSPR" and not is_subtask:
        print(
            "{key} is an open source pull request, and does not need to be processed.".format(
                key=issue_key
            ),
            file=sys.stderr,
        )
        return False

    user_url = URLObject(issue["fields"]["creator"]["self"])
    user_url = user_url.set_query_param("expand", "groups")

    user_resp = jira_get(user_url)
    if not user_resp.ok:
        raise requests.exceptions.RequestException(user_resp.text)

    user = user_resp.json()
    user_group_map = {g["name"]: g["self"] for g in user["groups"]["items"]}
    user_groups = set(user_group_map)

    exempt_groups = {
        # group name: set of projects that they can create non-triage issues
        "edx-employees": {"ALL"},
        "clarice": {"MOB"},
        "bnotions": {"MOB"},
        "opencraft": {"SOL"},
    }
    for user_group in user_groups:
        if user_group not in exempt_groups:
            continue
        exempt_projects = exempt_groups[user_group]
        if "ALL" in exempt_projects:
            return True
        if project_key in exempt_projects:
            return True

    return False
Ejemplo n.º 12
0
def _join_path(url, path):
    _url = URL(url)
    path = URL(path)
    if path.path:
        _url = _url.add_path(path.path)
    if path.query:
        _url = _url.with_query(path.query)
    return _url
Ejemplo n.º 13
0
    def test_with_auth_with_two_args_replaces_whole_auth_string_with_username_and_password(self):
        # Replaces username-only auth string
        url = URLObject('https://[email protected]/')
        assert url.with_auth('zack', '1234') == 'https://*****:*****@github.com/'

        # Replaces username and password.
        url = URLObject('https://*****:*****@github.com/')
        assert url.with_auth('zack', '1234') == 'https://*****:*****@github.com/'
Ejemplo n.º 14
0
    def clean_url(self):
        url = URLObject(self.cleaned_data["url"])

        # URLObject doesn't handle ipv6 very well yet. In the meantime, ...
        if url.netloc.count(":") > 3:
            raise forms.ValidationError(_("Enter a valid URL."))

        URLValidator()(url.without_auth())
        if url.scheme not in ["http", "https"]:
            raise forms.ValidationError(
                _("Invalid URL scheme: '%s'. Only HTTP and HTTPS are " "supported.") % url.scheme
            )

        if url.netloc.hostname in ["localhost", "127.0.0.1", "::1"]:
            raise forms.ValidationError(_("Enter a valid URL."))

        try:
            validate_ipv46_address(url.netloc.hostname)
        except forms.ValidationError:
            pass
        else:
            raise forms.ValidationError(_("Enter a valid URL."))

        existing = self.user.feeds.filter(url=url)
        if self.instance is not None:
            existing = existing.exclude(pk=self.instance.pk)

        if existing.exists():
            raise forms.ValidationError(_("It seems you're already subscribed to this feed."))

        auth = None
        if url.auth != (None, None):
            auth = url.auth

        # Check this is actually a feed
        with user_lock("feed_check", self.user.pk, timeout=30):
            headers = {"User-Agent": USER_AGENT % "checking feed", "Accept": feedparser.ACCEPT_HEADER}
            try:
                response = requests.get(six.text_type(url.without_auth()), headers=headers, timeout=10, auth=auth)
            except Exception:
                if "SENTRY_DSN" in os.environ:
                    client = Client()
                    client.captureException()
                raise forms.ValidationError(_("Error fetching the feed."))
            if response.status_code != 200:
                raise forms.ValidationError(_("Invalid response code from URL: " "HTTP %s.") % response.status_code)
        try:
            parsed = feedparser.parse(response.content)
        except Exception:
            raise forms.ValidationError(_("Error parsing the feed."))
        if not is_feed(parsed):
            raise forms.ValidationError(_("This URL doesn't seem to be a valid feed."))
        self.cleaned_data["title"] = parsed.feed.title
        # Cache this in case update_favicon needs it and it's not in the
        # scheduler data yet.
        if hasattr(parsed.feed, "link"):
            cache.set(u"feed_link:{0}".format(url), parsed.feed.link, 600)
        return url
Ejemplo n.º 15
0
    def authorized(self):
        if "next" in request.args:
            next_url = request.args["next"]
        elif self.redirect_url:
            next_url = self.redirect_url
        elif self.redirect_to:
            next_url = url_for(self.redirect_to)
        else:
            next_url = "/"

        # check for error in request args
        error = request.args.get("error")
        if error:
            error_desc = request.args.get("error_description")
            error_uri = request.args.get("error_uri")
            log.warning(
                "OAuth 2 authorization error: %s description: %s uri: %s",
                error, error_desc, error_uri,
            )
            oauth_error.send(self,
                error=error, error_description=error_desc, error_uri=error_uri,
            )
            return redirect(next_url)

        state_key = "{bp.name}_oauth_state".format(bp=self)
        self.session._state = flask.session[state_key]
        del flask.session[state_key]

        secure = request.is_secure or request.headers.get("X-Forwarded-Proto", "http") == "https"
        self.session.redirect_uri = url_for(
            ".authorized", next=request.args.get('next'), _external=True,
            _scheme="https" if secure else "http",
        )

        url = URLObject(request.url)
        if request.headers.get("X-Forwarded-Proto", "http") == "https":
            url = url.with_scheme("https")
        try:
            token = self.session.fetch_token(
                self.token_url,
                authorization_response=url,
                client_secret=self.client_secret,
                **self.token_url_params
            )
        except MissingCodeError as e:
            e.args = (
                e.args[0],
                "The redirect request did not contain the expected parameters. Instead I got: {}".format(
                    json.dumps(request.args)
                )
            )
            raise
        results = oauth_authorized.send(self, token=token) or []
        if not any(ret == False for func, ret in results):
            self.token = token
        return redirect(next_url)
Ejemplo n.º 16
0
 def test_set_query_params_with_multiple_values_adds_or_replaces_the_same_parameter_multiple_times(self):
     assert (self.url.set_query_params({'spam': ['bar', 'baz']}) ==
             'https://github.com/zacharyvoase/urlobject?spam=bar&spam=baz#foo')
     assert (self.url.set_query_params({'foo': ['bar', 'baz']}) ==
             'https://github.com/zacharyvoase/urlobject?spam=eggs&foo=bar&foo=baz#foo')
     # Ensure it removes all appearances of an existing name before adding
     # the new ones.
     url = URLObject('https://github.com/zacharyvoase/urlobject?foo=bar&foo=baz#foo')
     assert (url.set_query_params({'foo': ['spam', 'ham']}) ==
             'https://github.com/zacharyvoase/urlobject?foo=spam&foo=ham#foo')
Ejemplo n.º 17
0
def urls_are_equal(url1, url2):
    """
    Compare to URLs for equality, ignoring the ordering of non-ordered elements.
    """
    url1 = URLObject(url1)
    url2 = URLObject(url2)
    return (
        url1.without_query() == url2.without_query() and
        url1.query_multi_dict == url2.query_multi_dict
    )
Ejemplo n.º 18
0
def make_jira_blueprint(consumer_key, rsa_key, base_url,
                        redirect_url=None, redirect_to=None,
                        login_url=None, authorized_url=None):
    """
    Make a blueprint for authenticating with JIRA using OAuth 1.

    Args:
        consumer_key (str): The consumer key for your Application Link on JIRA
        rsa_key (str or path): The RSA private key for your Application Link
            on JIRA. This can be the contents of the key as a string, or a path
            to the key file on disk.
        base_url (str): The base URL of your JIRA installation. For example,
            for Atlassian's hosted OnDemand JIRA, the base_url would be
            ``https://jira.atlassian.com``
        redirect_url (str): the URL to redirect to after the authentication
            dance is complete
        redirect_to (str): if ``redirect_url`` is not defined, the name of the
            view to redirect to after the authentication dance is complete.
            The actual URL will be determined by :func:`flask.url_for`
        login_url (str, optional): the URL path for the ``login`` view.
            Defaults to ``/jira``
        authorized_url (str, optional): the URL path for the ``authorized`` view.
            Defaults to ``/jira/authorized``.

    :rtype: :class:`~flask_dance.consumer.OAuth1ConsumerBlueprint`
    :returns: A :ref:`blueprint <flask:blueprints>` to attach to your Flask app.
    """
    if os.path.isfile(rsa_key):
        with open(rsa_key) as f:
            rsa_key = f.read()
    base_url = URLObject(base_url)

    jira_bp = OAuth1ConsumerBlueprint("jira", __name__,
        client_key=consumer_key,
        rsa_key=rsa_key,
        signature_method=SIGNATURE_RSA,
        base_url=base_url,
        request_token_url=base_url.relative("plugins/servlet/oauth/request-token"),
        access_token_url=base_url.relative("plugins/servlet/oauth/access-token"),
        authorization_url=base_url.relative("plugins/servlet/oauth/authorize"),
        redirect_url=redirect_url,
        redirect_to=redirect_to,
        login_url=login_url,
        authorized_url=authorized_url,
    )
    jira_bp.session.headers["Content-Type"] = "application/json"

    @jira_bp.before_app_request
    def set_applocal_session():
        ctx = stack.top
        ctx.jira_oauth = jira_bp.session

    return jira_bp
Ejemplo n.º 19
0
    def test_query_dict_noseq(self):
        url = URLObject(scheme='http', host='www.google.com')
        url |= ('q', 'query')
        self.assertEqual(url.query_dict(seq=False), {u'q': u'query'})

        self.assertEqual(
            (url | ('q', 'another')).query_dict(seq=False),
            {u'q': u'another'})

        self.assertEqual(
            (url & ('q', 'another')).query_dict(seq=False),
            {u'q': u'another'})
Ejemplo n.º 20
0
    def url_with_page_number(self, page_number):
        """
        Constructs a url used for getting the next/previous urls
        """
        url = URLObject(self.request.get_full_path())
        url = url.set_query_param('page', str(page_number))

        limit = self.get_limit()
        if limit != self.limit:
            url = url.set_query_param('limit', str(limit))

        return url
Ejemplo n.º 21
0
    def test_query_list(self):
        url = URLObject(scheme='http', host='www.google.com')
        url |= ('q', 'query')
        self.assertEqual(url.query_list(), [(u'q', u'query')])

        self.assertEqual(
            (url | ('q', 'another')).query_list(),
            [(u'q', u'another')])

        self.assertEqual(
            (url & ('q', 'another')).query_list(),
            [(u'q', u'query'), (u'q', u'another')])
Ejemplo n.º 22
0
class API(object):

    def __init__(self, client, url):
        super(API, self).__init__()
        self.client = client
        self.url = URL(url)

    def call_function(self, name, params=None):
        resp = requests.post(
            self.url.add_path('api').add_path(name),
            data=self._serialize_params(params),
            headers={'Content-type': 'application/json'},
        )
        resp.raise_for_status()

        return self._normalize_return_value(resp)

    def get(self, path, raw=False):
        resp = requests.get(self.url.add_path(path))
        resp.raise_for_status()
        if raw:
            return resp.json()
        else:
            return self._normalize_return_value(resp)

    def _normalize_return_value(self, response):
        result = response.json()['result']
        if result is None:
            return None
        assert isinstance(result, dict) and 'type' in result
        return self.build_api_object(result)

    def build_api_object(self, result):
        return self._get_objtype(result)(self.client, result)

    def _get_objtype(self, json_object):
        typename = json_object['type']
        returned = _TYPES_BY_TYPENAME.get(typename)
        if returned is None:
            raise NotImplementedError()  # pragma: no cover
        return returned

    def _serialize_params(self, params):
        if params is None:
            params = {}

        returned = {}
        for param_name, param_value in iteritems(params):
            if param_value is NOTHING:
                continue
            returned[param_name] = param_value
        return json.dumps(returned)
Ejemplo n.º 23
0
def url_join(base, *paths):
    """
    Append `paths` to `base`. Path resets on each absolute path.

    Like os.path.join, but for URLs.
    """
    if not hasattr(base, 'add_path'):
        base = URLObject(base)

    for path in paths:
        path = URLPath(path)
        base = base.add_path(path)
    return base
Ejemplo n.º 24
0
 def __init__(self, args, context, tags, filters):
     self.args = args
     self.context = context
     self.tags = tags
     self.filters = filters
     self.autoescape = self.context.autoescape
     self.url = URLObject()
Ejemplo n.º 25
0
class App(object):

    def __init__(self, flask_app):
        super(App, self).__init__()
        self.flask_app = flask_app
        self.loopback = FlaskLoopback(self.flask_app)
        self.hostname = str(uuid1())
        self.url = URLObject("http://{0}".format(self.hostname))

    def activate(self):
        self.loopback.activate_address((self.hostname, 80))

    def deactivate(self):
        self.loopback.deactivate_address((self.hostname, 80))

    def get_page(self, page_size, page, path=None):
        if path is None:
            path = "objects"
        response = requests.get(self.url.add_path(path).set_query_param("page", str(page)).set_query_param("page_size", str(page_size)))
        response.raise_for_status()
        data = response.json()
        assert data["metadata"]["page"] == page
        assert data["metadata"]["page_size"] == page_size
        return data["result"]

    def get_all_paged(self, page_size, path=None):
        return list(itertools.chain.from_iterable(self.get_page(page_size, page, path=path) for page in range(1, int(self.num_objects / page_size) + 5)))
Ejemplo n.º 26
0
class OAuth2Session(BaseOAuth2Session):
    """
    A :class:`requests.Session` subclass that can do some special things:

    * lazy-loads OAuth2 tokens from the backend via the blueprint
    * handles OAuth2 authentication
      (from :class:`requests_oauthlib.OAuth2Session` superclass)
    * has a ``base_url`` property used for relative URL resolution
    """
    def __init__(self, blueprint=None, base_url=None, *args, **kwargs):
        super(OAuth2Session, self).__init__(*args, **kwargs)
        self.blueprint = blueprint
        self.base_url = URLObject(base_url)
        lazy.invalidate(self, "token")

    @lazy
    def token(self):
        return self.blueprint.token

    def request(self, method, url, data=None, headers=None, **kwargs):
        if self.base_url:
            url = self.base_url.relative(url)

        self._client.token = self.token
        if self.token:
            self._client._populate_attributes(self.token)

        return super(OAuth2Session, self).request(
            method=method, url=url, data=data, headers=headers, **kwargs
        )
Ejemplo n.º 27
0
class Connection(object):

    def __init__(self, url, email, token, name='default', version='v1',
                 cache=None):
        self.session = Connection._get_session(email, token)
        self.email = email
        self.base_url = URLObject(url)
        self.api_url = self.base_url.add_path_segment(version)
        self.cache = InMemoryCache() if cache is None else cache
        self.name = name

    def http_method(self, method, url, *args, **kwargs):
        """
        Send HTTP request with `method` to `url`.
        """
        method_fn = getattr(self.session, method)
        return method_fn(url, *args, **kwargs)

    def build_absolute_url(self, path):
        """
        Resolve relative `path` against this connection's API url.
        """
        return url_join(self.api_url, path)

    @staticmethod
    def _get_session(email, token):
        session = requests.Session()
        defaults = {
            'X-PW-Application': 'developer_api',
            'X-PW-AccessToken': token,
            'X-PW-UserEmail': email,
            'Accept': 'application/json',
            'Content-Type': 'application/json',
        }
        session.headers.update(defaults)
        return session

    def __getattr__(self, name):
        """
        Turn HTTP verbs into http_method calls so e.g. conn.get(...) works.

        Note that 'get' and 'delete' are special-cased to handle caching
        """
        methods = 'post', 'put', 'patch', 'options'
        if name in methods:
            return functools.partial(self.http_method, name)
        return super(Connection, self).__getattr__(name)

    def get(self, url, *args, **kwargs):
        cached = self.cache.get(url)
        if cached is None:
            cached = self.http_method('get', url, *args, **kwargs)
            self.cache.set(url, cached, max_age=seconds(minutes=5))
        return cached

    def delete(self, url, *args, **kwargs):
        resp = self.http_method('delete', url, *args, **kwargs)
        if resp.ok:
            self.cache.clear(url)
        return resp
Ejemplo n.º 28
0
class OAuth1Session(BaseOAuth1Session):
    """
    A :class:`requests.Session` subclass that can do some special things:

    * lazy-loads OAuth1 tokens from the backend via the blueprint
    * handles OAuth1 authentication
      (from :class:`requests_oauthlib.OAuth1Session` superclass)
    * has a ``base_url`` property used for relative URL resolution
    """
    def __init__(self, blueprint=None, base_url=None, *args, **kwargs):
        super(OAuth1Session, self).__init__(*args, **kwargs)
        self.blueprint = blueprint
        self.base_url = URLObject(base_url)

    @lazy
    def token(self):
        return self.blueprint.token

    def prepare_request(self, request):
        if self.base_url:
            request.url = self.base_url.relative(request.url)
        return super(OAuth1Session, self).prepare_request(request)

    def request(self, method, url, data=None, headers=None, **kwargs):
        t = self.token
        if t and "oauth_token" in t and "oauth_token_secret" in t:
            # This really, really violates the Law of Demeter, but
            # I don't see a better way to set these parameters. :(
            self.auth.client.resource_owner_key = to_unicode(t["oauth_token"])
            self.auth.client.resource_owner_secret = to_unicode(t["oauth_token_secret"])

        return super(OAuth1Session, self).request(
            method=method, url=url, data=data, headers=headers, **kwargs
        )
Ejemplo n.º 29
0
    def url_with_page_number(self, page_number):
        """
        Constructs a url used for getting the next/previous urls
        """
        if self.request.is_secure():
            protocol = 'https://'
        else:
            protocol = 'http://'
        url = URLObject(protocol+self.request.get_host()+self.request.get_full_path())
        url = url.set_query_param('page', str(page_number))

        limit = self.get_limit()
        if limit != self.limit:
            url = url.set_query_param('limit', str(limit))

        return url
Ejemplo n.º 30
0
class KeystoneAuth(object):
    def __init__(self, identity_url, username, password):
        self._identity_url = URLObject(identity_url)
        self._username = username
        self._password = password
        self._session = requests.Session()
        self._session.headers = {
            "content-type": "application/json",
            "accept": "application/json"
        }
        self._auth_token = None
        self._service_catalog = None

    def _perform_auth_request(self):
        auth_response = self._session.post(
            self._identity_url.add_path('tokens'),
            data=json.dumps({
                "auth": {
                    "passwordCredentials": {
                        "username": self._username,
                        "password": self._password
                    }
                },
                "tenantId": " "
            })
        )
        auth_response.raise_for_status()
        auth_body = auth_response.json()
        self._auth_token = auth_body["access"]["token"]["id"]
        self._service_catalog = ServiceCatalog(
            auth_body["access"]["serviceCatalog"]
        )

    def _handle_request_result(self, response, **kwargs):
        if response.status_code == 401:
            # We got an authentication failure, get a new token and try again.
            self._perform_auth_request()
            new_request = response.request.copy()
            new_request.headers["X-Auth-Token"] = self._auth_token
            new_response = response.connection.send(new_request, **kwargs)
            new_response.history.append(response)
            new_response.request = new_request
            return new_response
        else:
            return response

    def __call__(self, request):
        if self._auth_token is None:
            self._perform_auth_request()

        request.headers['X-Auth-Token'] = self._auth_token
        request.register_hook("response", self._handle_request_result)
        return request

    @property
    def service_catalog(self):
        if self._service_catalog is None:
            self._perform_auth_request()
        return self._service_catalog
Ejemplo n.º 31
0
    def _update_resource(self, cls, id, data, **kwargs):
        url = (URLObject(self.api_server).with_path("/{name}/{id}".format(
            name=cls.collection_name, id=id)).set_query_params(**kwargs))

        session = self._get_http_session(cls.api_root)

        converted_data = convert_datetimes_to_timestamps(
            data, cls.datetime_attrs)
        response = session.put(url, json=converted_data)

        result = _validate(response).json()
        return cls.create(self, **result)
Ejemplo n.º 32
0
def test_load_from_config():
    app = Flask(__name__)
    app.secret_key = "anything"
    app.config["SLACK_OAUTH_CLIENT_ID"] = "foo"
    app.config["SLACK_OAUTH_CLIENT_SECRET"] = "bar"
    slack_bp = make_slack_blueprint(redirect_to="index")
    app.register_blueprint(slack_bp)

    resp = app.test_client().get("/slack")
    url = resp.headers["Location"]
    client_id = URLObject(url).query.dict.get("client_id")
    assert client_id == "foo"
Ejemplo n.º 33
0
 def get_authorize_url(self, **kwargs) -> str:
     params = dict(
         client_id=self.client.client_id,  # type: ignore
         redirect_uri=self.client.redirect_uri,  # type: ignore
         scope=self.client.scope,  # type: ignore
         state=self.client.state,  # type: ignore
         response_type="code",
     )
     params.update(kwargs)
     return str(
         URLObject(self.client.authorize_url  # type: ignore
                   ).add_query_params(**params))
Ejemplo n.º 34
0
def anyway_server():
    server_thread = ServerThread()
    server_thread.start()
    sleep(0.1)

    url = URLObject("http://127.0.0.1:5000")
    response = requests.get(url)
    response.raise_for_status()

    yield url

    server_thread.shutdown()
Ejemplo n.º 35
0
 def __init__(self, client, url, runtoken):
     super(API, self).__init__()
     self.client = client
     self.url = URL(url)
     self.runtoken = runtoken
     self.session = requests.Session()
     self.session.headers.update({
         'X-Backslash-run-token': self.runtoken,
         'X-Backslash-client-version': BACKSLASH_CLIENT_VERSION,
     })
     self.call = CallProxy(self)
     self._cached_info = None
Ejemplo n.º 36
0
def get_pulls(owner_repo, labels=None, state="open", since=None, org=False):
    url = URLObject(
        "https://api.github.com/repos/{}/issues".format(owner_repo))
    if labels:
        url = url.set_query_param('labels', ",".join(labels))
    if since:
        url = url.set_query_param('since', since.isoformat())
    if state:
        url = url.set_query_param('state', state)
    url = url.set_query_param('sort', 'updated')

    org_fn = None
    if org:
        try:
            with open("people.yaml") as fpeople:
                people = yaml.load(fpeople)
            def_org = "other"
        except IOError:
            people = {}
            def_org = "---"

        def org_fn(issue):
            user_info = people.get(issue["user.login"])
            if not user_info:
                user_info = {"institution": "unsigned"}
            return user_info.get("institution", def_org)

    issues = JPullRequest.from_json(paginated_get(url), org_fn)
    if org:
        issues = sorted(issues, key=operator.itemgetter("org"))

    return issues
Ejemplo n.º 37
0
    def as_value(self, data, context):
        # The queries of the current URL, not using sequences here
        # since the order of sorting arguments matter
        url = URLObject(context['request'].get_full_path())
        queries = url.query.dict

        name, orderings = data['with'], data['by']
        query = self.find_query(queries.get(name), orderings, orderings[0])
        url = url.set_query_param(name, query)

        # If this isn't a block tag we probably only want the URL
        if not self._meta.block:
            return url

        label = self.nodelist.render(context)
        if not label.strip():
            raise TemplateSyntaxError("No label was specified")

        parts = []
        for part in query.split(','):
            part = part.strip()
            if part.startswith('-'):
                part = part.lstrip('-')
                # Translators: Used in title of descending sort fields
                text = _("'%(sort_field)s' (desc)")
            else:
                # Translators: Used in title of ascending sort fields
                text = _("'%(sort_field)s' (asc)")
            parts.append(text % {'sort_field': part})
        # Translators: Used for the link/form input title excluding the sort fields
        title = (_('Sort by: %(sort_fields)s') % {
            'sort_fields': get_text_list(parts, _('and'))
        })

        extra_context = dict(data,
                             title=title,
                             label=label,
                             url=url,
                             query=query)
        return render_to_string(self.using(data), extra_context, context)
Ejemplo n.º 38
0
def spawn_page_tasks_for_milestones(owner,
                                    repo,
                                    state="all",
                                    children=False,
                                    requestor_id=None,
                                    per_page=100):
    # acquire lock or fail (we're already in a transaction)
    lock_name = LOCK_TEMPLATE.format(owner=owner, repo=repo)
    existing = Mutex.query.get(lock_name)
    if existing:
        return False
    lock = Mutex(name=lock_name, user_id=requestor_id)
    db.session.add(lock)
    try:
        db.session.commit()
    except IntegrityError:
        return False
    else:
        logger.info("Lock {name} set by {requestor_id}".format(
            name=lock_name,
            requestor_id=requestor_id,
        ))

    milestone_list_url = ("/repos/{owner}/{repo}/pulls?"
                          "state={state}&per_page={per_page}").format(
                              owner=owner,
                              repo=repo,
                              state=state,
                              per_page=per_page,
                          )
    resp = fetch_url_from_github(
        milestone_list_url,
        method="HEAD",
        requestor_id=requestor_id,
    )
    last_page_url = URLObject(resp.links.get('last', {}).get('url', ""))
    last_page_num = int(last_page_url.query.dict.get('page', 1))
    g = group(
        sync_page_of_milestones.s(
            owner=owner,
            repo=repo,
            state=state,
            requestor_id=requestor_id,
            per_page=per_page,
            page=page,
        ) for page in xrange(1, last_page_num + 1))
    finisher = milestones_scanned.si(
        owner=owner,
        repo=repo,
        requestor_id=requestor_id,
    )
    return (g | finisher).delay()
Ejemplo n.º 39
0
def test_markers(app, show_fatal, show_severe, show_light, show_accurate,
                 show_approx, marker_counter):
    url = URLObject('/markers').set_query_params({
        "ne_lat": "32.085413468822",
        "ne_lng": "34.797736215591385",
        "sw_lat": "32.07001357040486",
        "sw_lng": "34.775548982620194",
        "zoom": "16",
        "thin_markers": "false",
        "start_date": "1104537600",
        "end_date": "1484697600",
        "show_fatal": show_fatal,
        "show_severe": show_severe,
        "show_light": show_light,
        "approx": show_approx,
        "accurate": show_accurate,
        "show_markers": "1",
        "show_accidents": "1",
        "show_rsa": "0",
        "show_discussions": "1",
        "show_urban": "3",
        "show_intersection": "3",
        "show_lane": "3",
        "show_day": "7",
        "show_holiday": "0",
        "show_time": "24",
        "start_time": "25",
        "end_time": "25",
        "weather": "0",
        "road": "0",
        "separation": "0",
        "surface": "0",
        "acctype": "0",
        "controlmeasure": "0",
        "district": "0",
        "case_type": "0"
    })

    rv = app.get(url)
    assert rv.status_code == http_client.OK
    assert rv.headers['Content-Type'] == 'application/json'

    resp = json.loads(_text_data(rv))

    marker_counter["markers"] += len(resp['markers'])

    for marker in resp['markers']:
        assert show_fatal or marker['accident_severity'] != 1
        assert show_severe or marker['accident_severity'] != 2
        assert show_light or marker['accident_severity'] != 3
        assert show_accurate or marker['location_accuracy'] != 1
        assert show_approx or marker['location_accuracy'] == 1
Ejemplo n.º 40
0
def get_referer(request, default='/'):
    '''
    Возвращает реферер не ссылающийся на текущую страницу
    '''
    original_referer = request.META.get('HTTP_REFERER')

    if not original_referer:
        return default

    url = URLObject(original_referer)

    # Проверяем нашего ли сайта домен
    host = '.%s' % url.hostname
    if url and not host.endswith('.%s' % settings.LOCAL.get('domain')):
        return default

    current_url = URLObject(request.get_full_path())

    # Проверяем не с этой же ли самой страницы реферер
    if url.path.strip('/') == current_url.path.strip('/'):
        return default
    return original_referer
Ejemplo n.º 41
0
def jira_group_members(groupname,
                       session=None,
                       start=0,
                       retries=3,
                       debug=False):
    """
    JIRA's group members API is horrible. This makes it easier to use.
    """
    session = session or requests.Session()
    url = URLObject("/rest/api/2/group").set_query_param(
        "groupname", groupname)
    more_results = True
    while more_results:
        end = start + 49  # max 50 users per page
        expand = "users[{start}:{end}]".format(start=start, end=end)
        result_url = url.set_query_param("expand", expand)
        for _ in xrange(retries):
            try:
                if debug:
                    print(result_url, file=sys.stderr)
                result_resp = session.get(result_url)
                result = result_resp.json()
                break
            except ValueError:
                continue
        if not result_resp.ok:
            raise requests.exceptions.RequestException(result_resp.text)
        result = result_resp.json()
        if not result:
            break
        users = result["users"]["items"]
        for user in users:
            yield user
        returned = len(users)
        total = result["users"]["size"]
        if start + returned < total:
            start += returned
        else:
            more_results = False
Ejemplo n.º 42
0
def test_client_authentication_url(api_client, api_url):
    expected = (
        URLObject(api_url).with_path("/oauth/authorize").set_query_params([
            ('login_hint', ''),
            ('state', ''),
            ('redirect_uri', '/redirect'),
            ('response_type', 'code'),
            ('client_id', 'None'),
            ('scope', 'email'),
        ]))
    actual = URLObject(api_client.authentication_url("/redirect"))
    assert urls_equal(expected, actual)

    actual2 = URLObject(
        api_client.authentication_url("/redirect", login_hint="hint"))
    expected2 = expected.set_query_param("login_hint", "hint")
    assert urls_equal(expected2, actual2)

    actual3 = URLObject(
        api_client.authentication_url("/redirect", state="confusion"))
    expected3 = expected.set_query_param("state", "confusion")
    assert urls_equal(expected3, actual3)
Ejemplo n.º 43
0
def test_client_authentication_url(api_client, api_url):
    expected = (
        URLObject(api_url).with_path("/oauth/authorize").set_query_params([
            ("login_hint", ""),
            ("state", ""),
            ("redirect_uri", "/redirect"),
            ("response_type", "code"),
            ("client_id", "None"),
            ("scopes", "email,calendar,contacts"),
        ]))
    actual = URLObject(api_client.authentication_url("/redirect"))
    assert urls_equal(expected, actual)

    actual2 = URLObject(
        api_client.authentication_url("/redirect", login_hint="hint"))
    expected2 = expected.set_query_param("login_hint", "hint")
    assert urls_equal(expected2, actual2)

    actual3 = URLObject(
        api_client.authentication_url("/redirect", state="confusion"))
    expected3 = expected.set_query_param("state", "confusion")
    assert urls_equal(expected3, actual3)
Ejemplo n.º 44
0
def modify_url_(url, operation, *args):
    """
    Враппер для функций модуля urlobject
    https://urlobject.readthedocs.org/en/latest/quickstart.html
    Назначение: разобрать текщий URL, поменять какую-то его часть и вернуть модифицированный URL в виде строки
    Например: modify_url(some_url, 'del_query_param', 'page') уберет пейджинг из запроса
    Возвращает URL без домена
    """
    if not operation:
        return url

    url = URLObject(url)

    if operation.endswith('_np'):
        url = url.del_query_param('page')
        operation = operation[0:-3]

    op = getattr(url, operation, None)
    if callable(op):
        return text_type(op(*args))
    raise Exception('%s is incorrect function name for urlobject.URLObject' %
                    operation)
Ejemplo n.º 45
0
class OAuth2SessionWithBaseURL(OAuth2Session):
    def __init__(self, base_url=None, *args, **kwargs):
        super(OAuth2SessionWithBaseURL, self).__init__(*args, **kwargs)
        self.base_url = URLObject(base_url)

    def request(self, method, url, data=None, headers=None, **kwargs):
        if self.base_url:
            url = self.base_url.relative(url)
        return super(OAuth2SessionWithBaseURL, self).request(method=method,
                                                             url=url,
                                                             data=data,
                                                             headers=headers,
                                                             **kwargs)
Ejemplo n.º 46
0
def test_non_zero_offset(mocked_responses, api_client, api_url):
    offset = random.randint(1, 1000)
    mocked_responses.add(
        responses.GET,
        api_url + '/events?in=Nylas&offset=' + str(offset),
        body='[]',
    )

    list(api_client.events.where({'in': 'Nylas', 'offset': offset}).items())
    url = mocked_responses.calls[-1].request.url
    query = URLObject(url).query_dict
    assert query['in'] == 'Nylas'
    assert query['offset'] == str(offset)
Ejemplo n.º 47
0
def list_restaurants():
    login_credentials = open("piotr").readlines()[0].strip()
    sb.login(login_credentials)
    wk = datetime.datetime.now().strftime("%A")
    restaurant_list = sb.list_restaurants(wk)
    if not restaurant_list:
        return "Looks like we don't order today or it's too late to do so!"
    buf = []
    for restaurant in restaurant_list:
        vlid = URLObject(restaurant['href']).query.dict['vendorLocationId']
        buf.append("<a href='%s'>%s</a>" %
                   (url_for("select_restaurants", id=vlid), restaurant.text))
    return "<br/>".join(buf)
Ejemplo n.º 48
0
    def test_with_auth_with_two_args_replaces_whole_auth_string_with_username_and_password(self):
        # Replaces username-only auth string
        url = URLObject(u'https://[email protected]/')
        assert url.with_auth('zack', '1234') == u'https://*****:*****@github.com/'

        # Replaces username and password.
        url = URLObject(u'https://*****:*****@github.com/')
        assert url.with_auth('zack', '1234') == u'https://*****:*****@github.com/'
Ejemplo n.º 49
0
def urls_are_equal(url1, url2):
    """
    Compare to URLs for equality, ignoring the ordering of non-ordered elements.
    """
    url1 = URLObject(url1)
    url2 = URLObject(url2)
    return (url1.without_query() == url2.without_query()
            and url1.query_multi_dict == url2.query_multi_dict)
Ejemplo n.º 50
0
class Mailboxer(object):
    def __init__(self, url):
        super(Mailboxer, self).__init__()
        self.url = URL(url).add_path("v2")

    def create_mailbox(self, address):
        self._post(self.url.add_path("mailboxes"), {"address": address})
        return Mailbox(self, address)

    def delete_mailbox(self, address):
        return self.get_mailbox(address).delete()

    def get_emails(self, address, unread=False):
        return self.get_mailbox(address).get_emails(unread)

    def get_mailboxes(self, **kwargs):
        return Query(self, self.url.add_path("mailboxes"), Mailbox, **kwargs)

    def get_mailbox(self, address):
        return Mailbox(self, address)

    def does_mailbox_exist(self, address):
        return Mailbox(self, address).exists()

    def _post(self, url, data):
        returned = requests.post(url,
                                 data=json.dumps(data),
                                 headers={"Content-type": "application/json"})
        returned.raise_for_status()
        return returned

    def _get_paged(self, url, obj):
        response = requests.get(url)
        response.raise_for_status()
        return [obj(data) for data in response.json()["result"]]

    def _mailbox_url(self, address):
        return self.url.add_path("mailboxes").add_path(address)
Ejemplo n.º 51
0
def test_login_url_forwarded_proto():
    app, _ = make_app()
    app.wsgi_app = ProxyFix(app.wsgi_app)
    with app.test_client() as client:
        resp = client.get(
            "/login/test-service",
            base_url="http://a.b.c",
            headers={"X-Forwarded-Proto": "https"},
            follow_redirects=False,
        )
    # check that we redirected the client with a https redirect_uri
    assert resp.status_code == 302
    location = URLObject(resp.headers["Location"])
    assert location.query_dict["redirect_uri"] == "https://a.b.c/login/test-service/authorized"
Ejemplo n.º 52
0
def get_quotes(title, quote_list, request, per_page=10, no_pages=False, query=False, tag=False):
	start = 0
	try: start = int(request.GET.get('start', ''))
	except: pass
	if start < 0: start = 0
	if query == False: template = loader.get_template('qdb/quotes.html')
	else: template = loader.get_template('qdb/search.html')
	quotes = quote_list[start:start+per_page]
	voted = []
	reported = []
	for i in range(len(quotes)): voted.append(request.session.get('voted', {}).get(str(quotes[i].id), False)); reported.append(request.session.get('reported', {}).get(str(quotes[i].id), False))
	url = URLObject(request.build_absolute_uri())
	context = {
		'title': title,
		'quote_list': list(zip(quotes, voted, reported)),
		'previous_page': False if start - per_page < 0 else url.with_query(url.query.set_param('start', str(start-per_page if start-per_page > 0 else 0))),
		'next_page': False if start + per_page >= len(quote_list) else url.with_query(url.query.set_param('start', str(start+per_page))),
		'no_pages': no_pages,
		'verified': request.session.get('verified', False)
	}
	if query != False: context['query'] = query
	if tag != False: context['tag'] = tag
	return HttpResponse(template.render(context, request))
Ejemplo n.º 53
0
def test_load_from_params():
    app = Flask(__name__)
    app.secret_key = "anything"
    app.config["ZOHO_OAUTH_CLIENT_ID"] = "foo"
    app.config["ZOHO_OAUTH_CLIENT_SECRET"] = "bar"
    zoho_bp = make_zoho_blueprint(
        client_id="not_foo",
        client_secret="not_bar"
    )
    app.register_blueprint(zoho_bp)
    resp = app.test_client().get("/zoho")
    url = resp.headers["Location"]
    client_id = URLObject(url).query.dict.get("client_id")
    assert client_id == "not_foo"
Ejemplo n.º 54
0
def make_jira_blueprint(consumer_key,
                        rsa_key,
                        base_url,
                        redirect_url=None,
                        redirect_to=None,
                        login_url=None,
                        authorized_url=None):
    if os.path.isfile(rsa_key):
        with open(rsa_key) as f:
            rsa_key = f.read()
    base_url = URLObject(base_url)

    jira_bp = OAuth1ConsumerBlueprint(
        "jira",
        __name__,
        client_key=consumer_key,
        rsa_key=rsa_key,
        signature_method=SIGNATURE_RSA,
        base_url=base_url,
        request_token_url=base_url.relative(
            "plugins/servlet/oauth/request-token"),
        access_token_url=base_url.relative(
            "plugins/servlet/oauth/access-token"),
        authorization_url=base_url.relative("plugins/servlet/oauth/authorize"),
        redirect_url=redirect_url,
        redirect_to=redirect_to,
        login_url=login_url,
        authorized_url=authorized_url,
    )
    jira_bp.session.headers["Content-Type"] = "application/json"

    @jira_bp.before_app_request
    def set_applocal_session():
        ctx = stack.top
        ctx.jira_oauth = jira_bp.session

    return jira_bp
Ejemplo n.º 55
0
 def start_requests(self):
     """
     Gets the spider started.
     If both `self.login_email` and `self.login_password` are set,
     this method generates a request to login with those credentials.
     Otherwise, this method generates a request to go to the "auto auth"
     page and get credentials from there. Either way, this method
     doesn't actually generate requests from `self.start_urls` -- that is
     handled by the `after_initial_login()` and `after_auto_auth()`
     methods.
     """
     if self.login_email and self.login_password:
         login_url = (URLObject("http://").with_hostname(
             self.domain).with_port(self.port).with_path(LOGIN_HTML_PATH))
         yield scrapy.Request(
             login_url,
             callback=self.after_initial_csrf,
         )
     else:
         self.logger.info(
             "email/password unset, fetching credentials via auto_auth")
         auth_url = (URLObject("http://").with_hostname(
             self.domain).with_port(
                 self.port).with_path(AUTO_AUTH_PATH).set_query_params(
                     staff='true',
                     course_id=self.course_key,
                 ))
         # make sure to request a parseable JSON response
         headers = {
             b"Accept": b"application/json",
         }
         yield scrapy.Request(
             auth_url,
             headers=headers,
             callback=self.after_auto_auth,
         )
Ejemplo n.º 56
0
 def __init__(self, url, db, layout, response_layout=None, **kwargs):
     '''
     :param url: The URL to access the FileMaker server. This should contain
         any authorization credentials. If a path is not provided (e.g. no
         trailing slash, like ``http://username:[email protected]``) then
         the default path of ``/fmi/xml/fmresultset.xml`` will be used.
     :param db: The database name to access (sets the ``-db`` parameter).
     :param layout: The layout to use (sets the ``-lay`` parameter).
     :param response_layout: (*Optional*) The layout to use (sets the
         ``-lay.response`` parameter).
     '''
     self.url = URLObject(url).without_auth()
     self.url = self.url.with_path(self.url.path
                                   or '/fmi/xml/fmresultset.xml')
     self.auth = URLObject(url).auth
     self.params = QueryDict('', mutable=True)
     self.dbparams = QueryDict('', mutable=True)
     self.dbparams.update({
         '-db': db,
         '-lay': layout,
     })
     if response_layout:
         self.dbparams['-lay.response'] = response_layout
     self.params['-max'] = '50'
Ejemplo n.º 57
0
 def __init__(self, url, strategy_name):
     self.headers = {
         'User-Agent': config.get_global_setting("user_agent"),
     }
     self.count = 0
     self.url = URLObject(url)
     self.strategy_name = strategy_name
     self.finish_count = 0
     self._cancel = False
     self.cond = threading.Condition()
     self.charset = config.auto_get_charset(url)
     self.vcode_len = config.get_websites_setting(self.url.hostname, "vcode_len")
     self.vcode_error = config.get_strategy_setting(self.strategy_name, "vcode_error")
     self.inputs_dict = config.get_strategy_inputs(self.strategy_name)
     results_list = config.get_strategy_setting(self.strategy_name, "results")
     self.results_select = [select for col_name, select in results_list]
     self.results_colname = [col_name for col_name, select in results_list]
     self.search_colname = None
     self.cookies = None
     self.hidden_data = None
     self.required_inputs = None
     self.img_url = None
     self.vcode = None
     self.df = None
Ejemplo n.º 58
0
    def handle_unexpected_redirect_to_login_page(self, response):
        """
        This method is called if the crawler has been unexpectedly logged out.
        If that happens, and the crawler requests a page that requires a
        logged-in user, the crawler will be redirected to a login page,
        with the originally-requested URL as the `next` query parameter.

        This method simply causes the crawler to log back in using the saved
        email and password credentials. We rely on the fact that the login
        page will redirect the user to the URL in the `next` query parameter
        if the login is successful -- this will allow the crawl to resume
        where it left off.

        This is method is very much like the `get_initial_login()` method,
        but the callback is `self.after_login` instead of
        `self.after_initial_login`.
        """
        next_url = URLObject(response.url).query_dict.get("next")
        login_url = (URLObject("http://").with_hostname(self.domain).with_port(
            self.port).with_path(LOGIN_API_PATH))
        if next_url:
            login_url = login_url.set_query_param("next", next_url)

        credentials = {
            "email": self.login_email,
            "password": self.login_password,
        }
        headers = {
            b"X-CSRFToken": get_csrf_token(response),
        }
        yield scrapy.FormRequest(
            login_url,
            formdata=credentials,
            headers=headers,
            callback=self.after_login,
        )
Ejemplo n.º 59
0
def test_offline_select_account_and_consent(make_app):
    app = make_app("foo",
                   "bar",
                   offline=True,
                   reprompt_consent=True,
                   reprompt_select_account=True)

    with app.test_client() as client:
        resp = client.get("/google",
                          base_url="https://a.b.c",
                          follow_redirects=False)
    assert resp.status_code == 302
    location = URLObject(resp.headers["Location"])
    assert location.query_dict["access_type"] == "offline"
    assert location.query_dict["prompt"] == "consent select_account"
Ejemplo n.º 60
0
def test_force_reapprove():
    app = Flask(__name__)
    app.secret_key = "forced"
    dropbox_bp = make_dropbox_blueprint("foo", "bar", force_reapprove=True)
    app.register_blueprint(dropbox_bp)

    with app.test_client() as client:
        resp = client.get(
            "/dropbox",
            base_url="https://a.b.c",
            follow_redirects=False,
        )
    # check that there is a `force_reapprove=true` query param in the redirect URL
    assert resp.status_code == 302
    location = URLObject(resp.headers["Location"])
    assert location.query_dict["force_reapprove"] == "true"