def jira_paginated_get(url, obj_name, session=None, start=0, retries=3, **fields):
"""
Like ``paginated_get``, but uses JIRA's conventions for a paginated API, which
are different from Github's conventions.
"""
session = session or requests.Session()
url = URLObject(url)
more_results = True
while more_results:
result_url = (
url.set_query_param("startAt", str(start))
.set_query_params(**fields)
)
for _ in xrange(retries):
try:
result_resp = session.get(result_url)
result = result_resp.json()
break
except ValueError:
continue
if not result_resp.ok:
raise requests.exceptions.RequestException(result)
result = result_resp.json()
for obj in result[obj_name]:
yield obj
returned = len(result[obj_name])
total = result["total"]
if start + returned < total:
start += returned
else:
more_results = False
def proxied(url):
url = URLObject(url)
netloc = url.netloc or settings.SERVER_NAME
cache = get_cache()
if netloc not in cache:
return url
return url.with_netloc(cache[netloc])
def process_url(url, tracking_id):
amazon_url = URLObject(url)
amazon_url = amazon_url.with_scheme('https')
amazon_url = amazon_url.set_query_param('tag', tracking_id)
return amazon_url
def process_request(self, request):
token = request.GET.get(self.param_name)
if not token:
return
redirect_url = URLObject(request.get_full_path())
redirect_url = redirect_url.del_query_param(self.param_name)
response = redirect(unicode(redirect_url))
try:
token_data = tempus_loads(token, max_age=self.max_age)
tempus = getattr(request, 'tempus', None)
if tempus:
current_tempus = tempus.copy()
current_tempus.update(token_data)
request.tempus = current_tempus
else:
request.tempus = token_data
except SignatureExpired:
value = self.__process_func(request, 'expired_func')
if value:
return value
except BadSignature:
value = self.__process_func(request, 'unsuccess_func')
if value:
return value
else:
value = self.__process_func(request, 'success_func')
if value:
return value
add_never_cache_headers(response)
return response
def test_multiple_parses_are_idempotent(self):
url = u'http://xn-hllo-bpa.com/path%20withspaces?query=es%25capes&foo=bar#frag%28withescapes%29'
parse1 = URLObject.parse(url)
self.assertEqual(unicode(url), unicode(parse1))
parse2 = URLObject.parse(unicode(parse1))
self.assertEqual(unicode(url), unicode(parse2))
self.assertEqual(unicode(parse1), unicode(parse2))
def jira_group_members(groupname, session=None, start=0, retries=3, debug=False):
"""
JIRA's group members API is horrible. This makes it easier to use.
"""
session = session or requests.Session()
url = URLObject("/rest/api/2/group").set_query_param("groupname", groupname)
more_results = True
while more_results:
end = start + 49 # max 50 users per page
expand = "users[{start}:{end}]".format(start=start, end=end)
result_url = url.set_query_param("expand", expand)
for _ in xrange(retries):
try:
if debug:
print(result_url, file=sys.stderr)
result_resp = session.get(result_url)
result = result_resp.json()
break
except ValueError:
continue
result_resp.raise_for_status()
result = result_resp.json()
if not result:
break
users = result["users"]["items"]
for user in users:
yield user
returned = len(users)
total = result["users"]["size"]
if start + returned < total:
start += returned
else:
more_results = False
def get_pulls(labels=None, state="open", since=None, org=False):
url = URLObject("https://api.github.com/repos/edx/edx-platform/issues")
if labels:
url = url.set_query_param('labels', ",".join(labels))
if since:
url = url.set_query_param('since', since.isoformat())
if state:
url = url.set_query_param('state', state)
url = url.set_query_param('sort', 'updated')
org_fn = None
if org:
try:
with open("mapping.yaml") as fmapping:
user_mapping = yaml.load(fmapping)
def_org = "other"
except IOError:
user_mapping = {}
def_org = "---"
def org_fn(issue):
return user_mapping.get(issue["user.login"], {}).get("institution", def_org)
issues = JPullRequest.from_json(paginated_get(url), org_fn)
if org:
issues = sorted(issues, key=operator.itemgetter("org"))
return issues
def show_pulls(jrep, labels=None, show_comments=False, state="open", since=None, org=False):
issues = get_pulls(labels, state, since, org)
category = None
for index, issue in enumerate(issues):
issue.finish_loading()
if issue.get("org") != category:
# new category! print category header
category = issue["org"]
print("-- {category} ----".format(category=category))
if 0:
import pprint
pprint.pprint(issue.obj)
print(issue.format(ISSUE_FMT))
if show_comments:
comments_url = URLObject(issue['comments_url'])
comments_url = comments_url.set_query_param("sort", "created")
comments_url = comments_url.set_query_param("direction", "desc")
comments = paginated_get(comments_url)
last_five_comments = reversed(more_itertools.take(5, comments))
for comment in last_five_comments:
print(comment.format(COMMENT_FMT))
# index is now set to the total number of pull requests
print()
print("{num} pull requests".format(num=index+1))
def get_pulls(owner_repo, labels=None, state="open", since=None, org=False):
url = URLObject("https://api.github.com/repos/{}/issues".format(owner_repo))
if labels:
url = url.set_query_param('labels', ",".join(labels))
if since:
url = url.set_query_param('since', since.isoformat())
if state:
url = url.set_query_param('state', state)
url = url.set_query_param('sort', 'updated')
org_fn = None
if org:
try:
with open("people.yaml") as fpeople:
people = yaml.load(fpeople)
def_org = "other"
except IOError:
people = {}
def_org = "---"
def org_fn(issue):
user_info = people.get(issue["user.login"])
if not user_info:
user_info = {"institution": "unsigned"}
return user_info.get("institution", def_org)
issues = JPullRequest.from_json(paginated_get(url), org_fn)
if org:
issues = sorted(issues, key=operator.itemgetter("org"))
return issues
def as_value(self, data, context):
# The queries of the current URL, not using sequences here
# since the order of sorting arguments matter
url = URLObject(context['request'].get_full_path())
queries = url.query.dict
name, orderings = data['with'], data['by']
query = self.find_query(queries.get(name), orderings, orderings[0])
url = url.set_query_param(name, query)
# If this isn't a block tag we probably only want the URL
if not self._meta.block:
return url
label = self.nodelist.render(context)
if not label.strip():
raise TemplateSyntaxError("No label was specified")
parts = []
for part in query.split(','):
part = part.strip()
if part.startswith('-'):
part = part.lstrip('-')
# Translators: Used in title of descending sort fields
text = _("'%(sort_field)s' (desc)")
else:
# Translators: Used in title of ascending sort fields
text = _("'%(sort_field)s' (asc)")
parts.append(text % {'sort_field': part})
# Translators: Used for the link/form input title excluding the sort fields
title = (_('Sort by: %(sort_fields)s') %
{'sort_fields': get_text_list(parts, _('and'))})
extra_context = dict(data, title=title, label=label, url=url, query=query)
return render_to_string(self.using(data), extra_context, context)
def should_transition(issue):
"""
Return a boolean indicating if the given issue should be transitioned
automatically from "Needs Triage" to an open status.
"""
issue_key = to_unicode(issue["key"])
issue_status = to_unicode(issue["fields"]["status"]["name"])
project_key = to_unicode(issue["fields"]["project"]["key"])
if issue_status != "Needs Triage":
print(
"{key} has status {status}, does not need to be processed".format(
key=issue_key, status=issue_status,
),
file=sys.stderr,
)
return False
# Open source pull requests do not skip Needs Triage.
# However, if someone creates a subtask on an OSPR issue, that subtasks
# might skip Needs Triage (it just follows the rest of the logic in this
# function.)
is_subtask = issue["fields"]["issuetype"]["subtask"]
if project_key == "OSPR" and not is_subtask:
print(
"{key} is an open source pull request, and does not need to be processed.".format(
key=issue_key
),
file=sys.stderr,
)
return False
user_url = URLObject(issue["fields"]["creator"]["self"])
user_url = user_url.set_query_param("expand", "groups")
user_resp = jira_get(user_url)
if not user_resp.ok:
raise requests.exceptions.RequestException(user_resp.text)
user = user_resp.json()
user_group_map = {g["name"]: g["self"] for g in user["groups"]["items"]}
user_groups = set(user_group_map)
exempt_groups = {
# group name: set of projects that they can create non-triage issues
"edx-employees": {"ALL"},
"clarice": {"MOB"},
"bnotions": {"MOB"},
"opencraft": {"SOL"},
}
for user_group in user_groups:
if user_group not in exempt_groups:
continue
exempt_projects = exempt_groups[user_group]
if "ALL" in exempt_projects:
return True
if project_key in exempt_projects:
return True
return False
def _join_path(url, path):
_url = URL(url)
path = URL(path)
if path.path:
_url = _url.add_path(path.path)
if path.query:
_url = _url.with_query(path.query)
return _url
def test_with_auth_with_two_args_replaces_whole_auth_string_with_username_and_password(self):
# Replaces username-only auth string
url = URLObject('https://[email protected]/')
assert url.with_auth('zack', '1234') == 'https://*****:*****@github.com/'
# Replaces username and password.
url = URLObject('https://*****:*****@github.com/')
assert url.with_auth('zack', '1234') == 'https://*****:*****@github.com/'
def clean_url(self):
url = URLObject(self.cleaned_data["url"])
# URLObject doesn't handle ipv6 very well yet. In the meantime, ...
if url.netloc.count(":") > 3:
raise forms.ValidationError(_("Enter a valid URL."))
URLValidator()(url.without_auth())
if url.scheme not in ["http", "https"]:
raise forms.ValidationError(
_("Invalid URL scheme: '%s'. Only HTTP and HTTPS are " "supported.") % url.scheme
)
if url.netloc.hostname in ["localhost", "127.0.0.1", "::1"]:
raise forms.ValidationError(_("Enter a valid URL."))
try:
validate_ipv46_address(url.netloc.hostname)
except forms.ValidationError:
pass
else:
raise forms.ValidationError(_("Enter a valid URL."))
existing = self.user.feeds.filter(url=url)
if self.instance is not None:
existing = existing.exclude(pk=self.instance.pk)
if existing.exists():
raise forms.ValidationError(_("It seems you're already subscribed to this feed."))
auth = None
if url.auth != (None, None):
auth = url.auth
# Check this is actually a feed
with user_lock("feed_check", self.user.pk, timeout=30):
headers = {"User-Agent": USER_AGENT % "checking feed", "Accept": feedparser.ACCEPT_HEADER}
try:
response = requests.get(six.text_type(url.without_auth()), headers=headers, timeout=10, auth=auth)
except Exception:
if "SENTRY_DSN" in os.environ:
client = Client()
client.captureException()
raise forms.ValidationError(_("Error fetching the feed."))
if response.status_code != 200:
raise forms.ValidationError(_("Invalid response code from URL: " "HTTP %s.") % response.status_code)
try:
parsed = feedparser.parse(response.content)
except Exception:
raise forms.ValidationError(_("Error parsing the feed."))
if not is_feed(parsed):
raise forms.ValidationError(_("This URL doesn't seem to be a valid feed."))
self.cleaned_data["title"] = parsed.feed.title
# Cache this in case update_favicon needs it and it's not in the
# scheduler data yet.
if hasattr(parsed.feed, "link"):
cache.set(u"feed_link:{0}".format(url), parsed.feed.link, 600)
return url
def authorized(self):
if "next" in request.args:
next_url = request.args["next"]
elif self.redirect_url:
next_url = self.redirect_url
elif self.redirect_to:
next_url = url_for(self.redirect_to)
else:
next_url = "/"
# check for error in request args
error = request.args.get("error")
if error:
error_desc = request.args.get("error_description")
error_uri = request.args.get("error_uri")
log.warning(
"OAuth 2 authorization error: %s description: %s uri: %s",
error, error_desc, error_uri,
)
oauth_error.send(self,
error=error, error_description=error_desc, error_uri=error_uri,
)
return redirect(next_url)
state_key = "{bp.name}_oauth_state".format(bp=self)
self.session._state = flask.session[state_key]
del flask.session[state_key]
secure = request.is_secure or request.headers.get("X-Forwarded-Proto", "http") == "https"
self.session.redirect_uri = url_for(
".authorized", next=request.args.get('next'), _external=True,
_scheme="https" if secure else "http",
)
url = URLObject(request.url)
if request.headers.get("X-Forwarded-Proto", "http") == "https":
url = url.with_scheme("https")
try:
token = self.session.fetch_token(
self.token_url,
authorization_response=url,
client_secret=self.client_secret,
**self.token_url_params
)
except MissingCodeError as e:
e.args = (
e.args[0],
"The redirect request did not contain the expected parameters. Instead I got: {}".format(
json.dumps(request.args)
)
)
raise
results = oauth_authorized.send(self, token=token) or []
if not any(ret == False for func, ret in results):
self.token = token
return redirect(next_url)
def test_set_query_params_with_multiple_values_adds_or_replaces_the_same_parameter_multiple_times(self):
assert (self.url.set_query_params({'spam': ['bar', 'baz']}) ==
'https://github.com/zacharyvoase/urlobject?spam=bar&spam=baz#foo')
assert (self.url.set_query_params({'foo': ['bar', 'baz']}) ==
'https://github.com/zacharyvoase/urlobject?spam=eggs&foo=bar&foo=baz#foo')
# Ensure it removes all appearances of an existing name before adding
# the new ones.
url = URLObject('https://github.com/zacharyvoase/urlobject?foo=bar&foo=baz#foo')
assert (url.set_query_params({'foo': ['spam', 'ham']}) ==
'https://github.com/zacharyvoase/urlobject?foo=spam&foo=ham#foo')
def urls_are_equal(url1, url2):
"""
Compare to URLs for equality, ignoring the ordering of non-ordered elements.
"""
url1 = URLObject(url1)
url2 = URLObject(url2)
return (
url1.without_query() == url2.without_query() and
url1.query_multi_dict == url2.query_multi_dict
)
def make_jira_blueprint(consumer_key, rsa_key, base_url,
redirect_url=None, redirect_to=None,
login_url=None, authorized_url=None):
"""
Make a blueprint for authenticating with JIRA using OAuth 1.
Args:
consumer_key (str): The consumer key for your Application Link on JIRA
rsa_key (str or path): The RSA private key for your Application Link
on JIRA. This can be the contents of the key as a string, or a path
to the key file on disk.
base_url (str): The base URL of your JIRA installation. For example,
for Atlassian's hosted OnDemand JIRA, the base_url would be
``https://jira.atlassian.com``
redirect_url (str): the URL to redirect to after the authentication
dance is complete
redirect_to (str): if ``redirect_url`` is not defined, the name of the
view to redirect to after the authentication dance is complete.
The actual URL will be determined by :func:`flask.url_for`
login_url (str, optional): the URL path for the ``login`` view.
Defaults to ``/jira``
authorized_url (str, optional): the URL path for the ``authorized`` view.
Defaults to ``/jira/authorized``.
:rtype: :class:`~flask_dance.consumer.OAuth1ConsumerBlueprint`
:returns: A :ref:`blueprint <flask:blueprints>` to attach to your Flask app.
"""
if os.path.isfile(rsa_key):
with open(rsa_key) as f:
rsa_key = f.read()
base_url = URLObject(base_url)
jira_bp = OAuth1ConsumerBlueprint("jira", __name__,
client_key=consumer_key,
rsa_key=rsa_key,
signature_method=SIGNATURE_RSA,
base_url=base_url,
request_token_url=base_url.relative("plugins/servlet/oauth/request-token"),
access_token_url=base_url.relative("plugins/servlet/oauth/access-token"),
authorization_url=base_url.relative("plugins/servlet/oauth/authorize"),
redirect_url=redirect_url,
redirect_to=redirect_to,
login_url=login_url,
authorized_url=authorized_url,
)
jira_bp.session.headers["Content-Type"] = "application/json"
@jira_bp.before_app_request
def set_applocal_session():
ctx = stack.top
ctx.jira_oauth = jira_bp.session
return jira_bp
def test_query_dict_noseq(self):
url = URLObject(scheme='http', host='www.google.com')
url |= ('q', 'query')
self.assertEqual(url.query_dict(seq=False), {u'q': u'query'})
self.assertEqual(
(url | ('q', 'another')).query_dict(seq=False),
{u'q': u'another'})
self.assertEqual(
(url & ('q', 'another')).query_dict(seq=False),
{u'q': u'another'})
def url_with_page_number(self, page_number):
"""
Constructs a url used for getting the next/previous urls
"""
url = URLObject(self.request.get_full_path())
url = url.set_query_param('page', str(page_number))
limit = self.get_limit()
if limit != self.limit:
url = url.set_query_param('limit', str(limit))
return url
def test_query_list(self):
url = URLObject(scheme='http', host='www.google.com')
url |= ('q', 'query')
self.assertEqual(url.query_list(), [(u'q', u'query')])
self.assertEqual(
(url | ('q', 'another')).query_list(),
[(u'q', u'another')])
self.assertEqual(
(url & ('q', 'another')).query_list(),
[(u'q', u'query'), (u'q', u'another')])
class API(object):
def __init__(self, client, url):
super(API, self).__init__()
self.client = client
self.url = URL(url)
def call_function(self, name, params=None):
resp = requests.post(
self.url.add_path('api').add_path(name),
data=self._serialize_params(params),
headers={'Content-type': 'application/json'},
)
resp.raise_for_status()
return self._normalize_return_value(resp)
def get(self, path, raw=False):
resp = requests.get(self.url.add_path(path))
resp.raise_for_status()
if raw:
return resp.json()
else:
return self._normalize_return_value(resp)
def _normalize_return_value(self, response):
result = response.json()['result']
if result is None:
return None
assert isinstance(result, dict) and 'type' in result
return self.build_api_object(result)
def build_api_object(self, result):
return self._get_objtype(result)(self.client, result)
def _get_objtype(self, json_object):
typename = json_object['type']
returned = _TYPES_BY_TYPENAME.get(typename)
if returned is None:
raise NotImplementedError() # pragma: no cover
return returned
def _serialize_params(self, params):
if params is None:
params = {}
returned = {}
for param_name, param_value in iteritems(params):
if param_value is NOTHING:
continue
returned[param_name] = param_value
return json.dumps(returned)
def url_join(base, *paths):
"""
Append `paths` to `base`. Path resets on each absolute path.
Like os.path.join, but for URLs.
"""
if not hasattr(base, 'add_path'):
base = URLObject(base)
for path in paths:
path = URLPath(path)
base = base.add_path(path)
return base
def __init__(self, args, context, tags, filters):
self.args = args
self.context = context
self.tags = tags
self.filters = filters
self.autoescape = self.context.autoescape
self.url = URLObject()
class App(object):
def __init__(self, flask_app):
super(App, self).__init__()
self.flask_app = flask_app
self.loopback = FlaskLoopback(self.flask_app)
self.hostname = str(uuid1())
self.url = URLObject("http://{0}".format(self.hostname))
def activate(self):
self.loopback.activate_address((self.hostname, 80))
def deactivate(self):
self.loopback.deactivate_address((self.hostname, 80))
def get_page(self, page_size, page, path=None):
if path is None:
path = "objects"
response = requests.get(self.url.add_path(path).set_query_param("page", str(page)).set_query_param("page_size", str(page_size)))
response.raise_for_status()
data = response.json()
assert data["metadata"]["page"] == page
assert data["metadata"]["page_size"] == page_size
return data["result"]
def get_all_paged(self, page_size, path=None):
return list(itertools.chain.from_iterable(self.get_page(page_size, page, path=path) for page in range(1, int(self.num_objects / page_size) + 5)))
class OAuth2Session(BaseOAuth2Session):
"""
A :class:`requests.Session` subclass that can do some special things:
* lazy-loads OAuth2 tokens from the backend via the blueprint
* handles OAuth2 authentication
(from :class:`requests_oauthlib.OAuth2Session` superclass)
* has a ``base_url`` property used for relative URL resolution
"""
def __init__(self, blueprint=None, base_url=None, *args, **kwargs):
super(OAuth2Session, self).__init__(*args, **kwargs)
self.blueprint = blueprint
self.base_url = URLObject(base_url)
lazy.invalidate(self, "token")
@lazy
def token(self):
return self.blueprint.token
def request(self, method, url, data=None, headers=None, **kwargs):
if self.base_url:
url = self.base_url.relative(url)
self._client.token = self.token
if self.token:
self._client._populate_attributes(self.token)
return super(OAuth2Session, self).request(
method=method, url=url, data=data, headers=headers, **kwargs
)
class Connection(object):
def __init__(self, url, email, token, name='default', version='v1',
cache=None):
self.session = Connection._get_session(email, token)
self.email = email
self.base_url = URLObject(url)
self.api_url = self.base_url.add_path_segment(version)
self.cache = InMemoryCache() if cache is None else cache
self.name = name
def http_method(self, method, url, *args, **kwargs):
"""
Send HTTP request with `method` to `url`.
"""
method_fn = getattr(self.session, method)
return method_fn(url, *args, **kwargs)
def build_absolute_url(self, path):
"""
Resolve relative `path` against this connection's API url.
"""
return url_join(self.api_url, path)
@staticmethod
def _get_session(email, token):
session = requests.Session()
defaults = {
'X-PW-Application': 'developer_api',
'X-PW-AccessToken': token,
'X-PW-UserEmail': email,
'Accept': 'application/json',
'Content-Type': 'application/json',
}
session.headers.update(defaults)
return session
def __getattr__(self, name):
"""
Turn HTTP verbs into http_method calls so e.g. conn.get(...) works.
Note that 'get' and 'delete' are special-cased to handle caching
"""
methods = 'post', 'put', 'patch', 'options'
if name in methods:
return functools.partial(self.http_method, name)
return super(Connection, self).__getattr__(name)
def get(self, url, *args, **kwargs):
cached = self.cache.get(url)
if cached is None:
cached = self.http_method('get', url, *args, **kwargs)
self.cache.set(url, cached, max_age=seconds(minutes=5))
return cached
def delete(self, url, *args, **kwargs):
resp = self.http_method('delete', url, *args, **kwargs)
if resp.ok:
self.cache.clear(url)
return resp
class OAuth1Session(BaseOAuth1Session):
"""
A :class:`requests.Session` subclass that can do some special things:
* lazy-loads OAuth1 tokens from the backend via the blueprint
* handles OAuth1 authentication
(from :class:`requests_oauthlib.OAuth1Session` superclass)
* has a ``base_url`` property used for relative URL resolution
"""
def __init__(self, blueprint=None, base_url=None, *args, **kwargs):
super(OAuth1Session, self).__init__(*args, **kwargs)
self.blueprint = blueprint
self.base_url = URLObject(base_url)
@lazy
def token(self):
return self.blueprint.token
def prepare_request(self, request):
if self.base_url:
request.url = self.base_url.relative(request.url)
return super(OAuth1Session, self).prepare_request(request)
def request(self, method, url, data=None, headers=None, **kwargs):
t = self.token
if t and "oauth_token" in t and "oauth_token_secret" in t:
# This really, really violates the Law of Demeter, but
# I don't see a better way to set these parameters. :(
self.auth.client.resource_owner_key = to_unicode(t["oauth_token"])
self.auth.client.resource_owner_secret = to_unicode(t["oauth_token_secret"])
return super(OAuth1Session, self).request(
method=method, url=url, data=data, headers=headers, **kwargs
)
def url_with_page_number(self, page_number):
"""
Constructs a url used for getting the next/previous urls
"""
if self.request.is_secure():
protocol = 'https://'
else:
protocol = 'http://'
url = URLObject(protocol+self.request.get_host()+self.request.get_full_path())
url = url.set_query_param('page', str(page_number))
limit = self.get_limit()
if limit != self.limit:
url = url.set_query_param('limit', str(limit))
return url
class KeystoneAuth(object):
def __init__(self, identity_url, username, password):
self._identity_url = URLObject(identity_url)
self._username = username
self._password = password
self._session = requests.Session()
self._session.headers = {
"content-type": "application/json",
"accept": "application/json"
}
self._auth_token = None
self._service_catalog = None
def _perform_auth_request(self):
auth_response = self._session.post(
self._identity_url.add_path('tokens'),
data=json.dumps({
"auth": {
"passwordCredentials": {
"username": self._username,
"password": self._password
}
},
"tenantId": " "
})
)
auth_response.raise_for_status()
auth_body = auth_response.json()
self._auth_token = auth_body["access"]["token"]["id"]
self._service_catalog = ServiceCatalog(
auth_body["access"]["serviceCatalog"]
)
def _handle_request_result(self, response, **kwargs):
if response.status_code == 401:
# We got an authentication failure, get a new token and try again.
self._perform_auth_request()
new_request = response.request.copy()
new_request.headers["X-Auth-Token"] = self._auth_token
new_response = response.connection.send(new_request, **kwargs)
new_response.history.append(response)
new_response.request = new_request
return new_response
else:
return response
def __call__(self, request):
if self._auth_token is None:
self._perform_auth_request()
request.headers['X-Auth-Token'] = self._auth_token
request.register_hook("response", self._handle_request_result)
return request
@property
def service_catalog(self):
if self._service_catalog is None:
self._perform_auth_request()
return self._service_catalog
def _update_resource(self, cls, id, data, **kwargs):
url = (URLObject(self.api_server).with_path("/{name}/{id}".format(
name=cls.collection_name, id=id)).set_query_params(**kwargs))
session = self._get_http_session(cls.api_root)
converted_data = convert_datetimes_to_timestamps(
data, cls.datetime_attrs)
response = session.put(url, json=converted_data)
result = _validate(response).json()
return cls.create(self, **result)
def test_load_from_config():
app = Flask(__name__)
app.secret_key = "anything"
app.config["SLACK_OAUTH_CLIENT_ID"] = "foo"
app.config["SLACK_OAUTH_CLIENT_SECRET"] = "bar"
slack_bp = make_slack_blueprint(redirect_to="index")
app.register_blueprint(slack_bp)
resp = app.test_client().get("/slack")
url = resp.headers["Location"]
client_id = URLObject(url).query.dict.get("client_id")
assert client_id == "foo"
def get_authorize_url(self, **kwargs) -> str:
params = dict(
client_id=self.client.client_id, # type: ignore
redirect_uri=self.client.redirect_uri, # type: ignore
scope=self.client.scope, # type: ignore
state=self.client.state, # type: ignore
response_type="code",
)
params.update(kwargs)
return str(
URLObject(self.client.authorize_url # type: ignore
).add_query_params(**params))
def anyway_server():
server_thread = ServerThread()
server_thread.start()
sleep(0.1)
url = URLObject("http://127.0.0.1:5000")
response = requests.get(url)
response.raise_for_status()
yield url
server_thread.shutdown()
def __init__(self, client, url, runtoken):
super(API, self).__init__()
self.client = client
self.url = URL(url)
self.runtoken = runtoken
self.session = requests.Session()
self.session.headers.update({
'X-Backslash-run-token': self.runtoken,
'X-Backslash-client-version': BACKSLASH_CLIENT_VERSION,
})
self.call = CallProxy(self)
self._cached_info = None
def get_pulls(owner_repo, labels=None, state="open", since=None, org=False):
url = URLObject(
"https://api.github.com/repos/{}/issues".format(owner_repo))
if labels:
url = url.set_query_param('labels', ",".join(labels))
if since:
url = url.set_query_param('since', since.isoformat())
if state:
url = url.set_query_param('state', state)
url = url.set_query_param('sort', 'updated')
org_fn = None
if org:
try:
with open("people.yaml") as fpeople:
people = yaml.load(fpeople)
def_org = "other"
except IOError:
people = {}
def_org = "---"
def org_fn(issue):
user_info = people.get(issue["user.login"])
if not user_info:
user_info = {"institution": "unsigned"}
return user_info.get("institution", def_org)
issues = JPullRequest.from_json(paginated_get(url), org_fn)
if org:
issues = sorted(issues, key=operator.itemgetter("org"))
return issues
def as_value(self, data, context):
# The queries of the current URL, not using sequences here
# since the order of sorting arguments matter
url = URLObject(context['request'].get_full_path())
queries = url.query.dict
name, orderings = data['with'], data['by']
query = self.find_query(queries.get(name), orderings, orderings[0])
url = url.set_query_param(name, query)
# If this isn't a block tag we probably only want the URL
if not self._meta.block:
return url
label = self.nodelist.render(context)
if not label.strip():
raise TemplateSyntaxError("No label was specified")
parts = []
for part in query.split(','):
part = part.strip()
if part.startswith('-'):
part = part.lstrip('-')
# Translators: Used in title of descending sort fields
text = _("'%(sort_field)s' (desc)")
else:
# Translators: Used in title of ascending sort fields
text = _("'%(sort_field)s' (asc)")
parts.append(text % {'sort_field': part})
# Translators: Used for the link/form input title excluding the sort fields
title = (_('Sort by: %(sort_fields)s') % {
'sort_fields': get_text_list(parts, _('and'))
})
extra_context = dict(data,
title=title,
label=label,
url=url,
query=query)
return render_to_string(self.using(data), extra_context, context)
def spawn_page_tasks_for_milestones(owner,
repo,
state="all",
children=False,
requestor_id=None,
per_page=100):
# acquire lock or fail (we're already in a transaction)
lock_name = LOCK_TEMPLATE.format(owner=owner, repo=repo)
existing = Mutex.query.get(lock_name)
if existing:
return False
lock = Mutex(name=lock_name, user_id=requestor_id)
db.session.add(lock)
try:
db.session.commit()
except IntegrityError:
return False
else:
logger.info("Lock {name} set by {requestor_id}".format(
name=lock_name,
requestor_id=requestor_id,
))
milestone_list_url = ("/repos/{owner}/{repo}/pulls?"
"state={state}&per_page={per_page}").format(
owner=owner,
repo=repo,
state=state,
per_page=per_page,
)
resp = fetch_url_from_github(
milestone_list_url,
method="HEAD",
requestor_id=requestor_id,
)
last_page_url = URLObject(resp.links.get('last', {}).get('url', ""))
last_page_num = int(last_page_url.query.dict.get('page', 1))
g = group(
sync_page_of_milestones.s(
owner=owner,
repo=repo,
state=state,
requestor_id=requestor_id,
per_page=per_page,
page=page,
) for page in xrange(1, last_page_num + 1))
finisher = milestones_scanned.si(
owner=owner,
repo=repo,
requestor_id=requestor_id,
)
return (g | finisher).delay()
def test_markers(app, show_fatal, show_severe, show_light, show_accurate,
show_approx, marker_counter):
url = URLObject('/markers').set_query_params({
"ne_lat": "32.085413468822",
"ne_lng": "34.797736215591385",
"sw_lat": "32.07001357040486",
"sw_lng": "34.775548982620194",
"zoom": "16",
"thin_markers": "false",
"start_date": "1104537600",
"end_date": "1484697600",
"show_fatal": show_fatal,
"show_severe": show_severe,
"show_light": show_light,
"approx": show_approx,
"accurate": show_accurate,
"show_markers": "1",
"show_accidents": "1",
"show_rsa": "0",
"show_discussions": "1",
"show_urban": "3",
"show_intersection": "3",
"show_lane": "3",
"show_day": "7",
"show_holiday": "0",
"show_time": "24",
"start_time": "25",
"end_time": "25",
"weather": "0",
"road": "0",
"separation": "0",
"surface": "0",
"acctype": "0",
"controlmeasure": "0",
"district": "0",
"case_type": "0"
})
rv = app.get(url)
assert rv.status_code == http_client.OK
assert rv.headers['Content-Type'] == 'application/json'
resp = json.loads(_text_data(rv))
marker_counter["markers"] += len(resp['markers'])
for marker in resp['markers']:
assert show_fatal or marker['accident_severity'] != 1
assert show_severe or marker['accident_severity'] != 2
assert show_light or marker['accident_severity'] != 3
assert show_accurate or marker['location_accuracy'] != 1
assert show_approx or marker['location_accuracy'] == 1
def get_referer(request, default='/'):
'''
Возвращает реферер не ссылающийся на текущую страницу
'''
original_referer = request.META.get('HTTP_REFERER')
if not original_referer:
return default
url = URLObject(original_referer)
# Проверяем нашего ли сайта домен
host = '.%s' % url.hostname
if url and not host.endswith('.%s' % settings.LOCAL.get('domain')):
return default
current_url = URLObject(request.get_full_path())
# Проверяем не с этой же ли самой страницы реферер
if url.path.strip('/') == current_url.path.strip('/'):
return default
return original_referer
def jira_group_members(groupname,
session=None,
start=0,
retries=3,
debug=False):
"""
JIRA's group members API is horrible. This makes it easier to use.
"""
session = session or requests.Session()
url = URLObject("/rest/api/2/group").set_query_param(
"groupname", groupname)
more_results = True
while more_results:
end = start + 49 # max 50 users per page
expand = "users[{start}:{end}]".format(start=start, end=end)
result_url = url.set_query_param("expand", expand)
for _ in xrange(retries):
try:
if debug:
print(result_url, file=sys.stderr)
result_resp = session.get(result_url)
result = result_resp.json()
break
except ValueError:
continue
if not result_resp.ok:
raise requests.exceptions.RequestException(result_resp.text)
result = result_resp.json()
if not result:
break
users = result["users"]["items"]
for user in users:
yield user
returned = len(users)
total = result["users"]["size"]
if start + returned < total:
start += returned
else:
more_results = False
def test_client_authentication_url(api_client, api_url):
expected = (
URLObject(api_url).with_path("/oauth/authorize").set_query_params([
('login_hint', ''),
('state', ''),
('redirect_uri', '/redirect'),
('response_type', 'code'),
('client_id', 'None'),
('scope', 'email'),
]))
actual = URLObject(api_client.authentication_url("/redirect"))
assert urls_equal(expected, actual)
actual2 = URLObject(
api_client.authentication_url("/redirect", login_hint="hint"))
expected2 = expected.set_query_param("login_hint", "hint")
assert urls_equal(expected2, actual2)
actual3 = URLObject(
api_client.authentication_url("/redirect", state="confusion"))
expected3 = expected.set_query_param("state", "confusion")
assert urls_equal(expected3, actual3)
def test_client_authentication_url(api_client, api_url):
expected = (
URLObject(api_url).with_path("/oauth/authorize").set_query_params([
("login_hint", ""),
("state", ""),
("redirect_uri", "/redirect"),
("response_type", "code"),
("client_id", "None"),
("scopes", "email,calendar,contacts"),
]))
actual = URLObject(api_client.authentication_url("/redirect"))
assert urls_equal(expected, actual)
actual2 = URLObject(
api_client.authentication_url("/redirect", login_hint="hint"))
expected2 = expected.set_query_param("login_hint", "hint")
assert urls_equal(expected2, actual2)
actual3 = URLObject(
api_client.authentication_url("/redirect", state="confusion"))
expected3 = expected.set_query_param("state", "confusion")
assert urls_equal(expected3, actual3)
def modify_url_(url, operation, *args):
"""
Враппер для функций модуля urlobject
https://urlobject.readthedocs.org/en/latest/quickstart.html
Назначение: разобрать текщий URL, поменять какую-то его часть и вернуть модифицированный URL в виде строки
Например: modify_url(some_url, 'del_query_param', 'page') уберет пейджинг из запроса
Возвращает URL без домена
"""
if not operation:
return url
url = URLObject(url)
if operation.endswith('_np'):
url = url.del_query_param('page')
operation = operation[0:-3]
op = getattr(url, operation, None)
if callable(op):
return text_type(op(*args))
raise Exception('%s is incorrect function name for urlobject.URLObject' %
operation)
class OAuth2SessionWithBaseURL(OAuth2Session):
def __init__(self, base_url=None, *args, **kwargs):
super(OAuth2SessionWithBaseURL, self).__init__(*args, **kwargs)
self.base_url = URLObject(base_url)
def request(self, method, url, data=None, headers=None, **kwargs):
if self.base_url:
url = self.base_url.relative(url)
return super(OAuth2SessionWithBaseURL, self).request(method=method,
url=url,
data=data,
headers=headers,
**kwargs)
def test_non_zero_offset(mocked_responses, api_client, api_url):
offset = random.randint(1, 1000)
mocked_responses.add(
responses.GET,
api_url + '/events?in=Nylas&offset=' + str(offset),
body='[]',
)
list(api_client.events.where({'in': 'Nylas', 'offset': offset}).items())
url = mocked_responses.calls[-1].request.url
query = URLObject(url).query_dict
assert query['in'] == 'Nylas'
assert query['offset'] == str(offset)
def list_restaurants():
login_credentials = open("piotr").readlines()[0].strip()
sb.login(login_credentials)
wk = datetime.datetime.now().strftime("%A")
restaurant_list = sb.list_restaurants(wk)
if not restaurant_list:
return "Looks like we don't order today or it's too late to do so!"
buf = []
for restaurant in restaurant_list:
vlid = URLObject(restaurant['href']).query.dict['vendorLocationId']
buf.append("<a href='%s'>%s</a>" %
(url_for("select_restaurants", id=vlid), restaurant.text))
return "<br/>".join(buf)
def test_with_auth_with_two_args_replaces_whole_auth_string_with_username_and_password(self):
# Replaces username-only auth string
url = URLObject(u'https://[email protected]/')
assert url.with_auth('zack', '1234') == u'https://*****:*****@github.com/'
# Replaces username and password.
url = URLObject(u'https://*****:*****@github.com/')
assert url.with_auth('zack', '1234') == u'https://*****:*****@github.com/'
def urls_are_equal(url1, url2):
"""
Compare to URLs for equality, ignoring the ordering of non-ordered elements.
"""
url1 = URLObject(url1)
url2 = URLObject(url2)
return (url1.without_query() == url2.without_query()
and url1.query_multi_dict == url2.query_multi_dict)
class Mailboxer(object):
def __init__(self, url):
super(Mailboxer, self).__init__()
self.url = URL(url).add_path("v2")
def create_mailbox(self, address):
self._post(self.url.add_path("mailboxes"), {"address": address})
return Mailbox(self, address)
def delete_mailbox(self, address):
return self.get_mailbox(address).delete()
def get_emails(self, address, unread=False):
return self.get_mailbox(address).get_emails(unread)
def get_mailboxes(self, **kwargs):
return Query(self, self.url.add_path("mailboxes"), Mailbox, **kwargs)
def get_mailbox(self, address):
return Mailbox(self, address)
def does_mailbox_exist(self, address):
return Mailbox(self, address).exists()
def _post(self, url, data):
returned = requests.post(url,
data=json.dumps(data),
headers={"Content-type": "application/json"})
returned.raise_for_status()
return returned
def _get_paged(self, url, obj):
response = requests.get(url)
response.raise_for_status()
return [obj(data) for data in response.json()["result"]]
def _mailbox_url(self, address):
return self.url.add_path("mailboxes").add_path(address)
def test_login_url_forwarded_proto():
app, _ = make_app()
app.wsgi_app = ProxyFix(app.wsgi_app)
with app.test_client() as client:
resp = client.get(
"/login/test-service",
base_url="http://a.b.c",
headers={"X-Forwarded-Proto": "https"},
follow_redirects=False,
)
# check that we redirected the client with a https redirect_uri
assert resp.status_code == 302
location = URLObject(resp.headers["Location"])
assert location.query_dict["redirect_uri"] == "https://a.b.c/login/test-service/authorized"
def get_quotes(title, quote_list, request, per_page=10, no_pages=False, query=False, tag=False):
start = 0
try: start = int(request.GET.get('start', ''))
except: pass
if start < 0: start = 0
if query == False: template = loader.get_template('qdb/quotes.html')
else: template = loader.get_template('qdb/search.html')
quotes = quote_list[start:start+per_page]
voted = []
reported = []
for i in range(len(quotes)): voted.append(request.session.get('voted', {}).get(str(quotes[i].id), False)); reported.append(request.session.get('reported', {}).get(str(quotes[i].id), False))
url = URLObject(request.build_absolute_uri())
context = {
'title': title,
'quote_list': list(zip(quotes, voted, reported)),
'previous_page': False if start - per_page < 0 else url.with_query(url.query.set_param('start', str(start-per_page if start-per_page > 0 else 0))),
'next_page': False if start + per_page >= len(quote_list) else url.with_query(url.query.set_param('start', str(start+per_page))),
'no_pages': no_pages,
'verified': request.session.get('verified', False)
}
if query != False: context['query'] = query
if tag != False: context['tag'] = tag
return HttpResponse(template.render(context, request))
def test_load_from_params():
app = Flask(__name__)
app.secret_key = "anything"
app.config["ZOHO_OAUTH_CLIENT_ID"] = "foo"
app.config["ZOHO_OAUTH_CLIENT_SECRET"] = "bar"
zoho_bp = make_zoho_blueprint(
client_id="not_foo",
client_secret="not_bar"
)
app.register_blueprint(zoho_bp)
resp = app.test_client().get("/zoho")
url = resp.headers["Location"]
client_id = URLObject(url).query.dict.get("client_id")
assert client_id == "not_foo"
def make_jira_blueprint(consumer_key,
rsa_key,
base_url,
redirect_url=None,
redirect_to=None,
login_url=None,
authorized_url=None):
if os.path.isfile(rsa_key):
with open(rsa_key) as f:
rsa_key = f.read()
base_url = URLObject(base_url)
jira_bp = OAuth1ConsumerBlueprint(
"jira",
__name__,
client_key=consumer_key,
rsa_key=rsa_key,
signature_method=SIGNATURE_RSA,
base_url=base_url,
request_token_url=base_url.relative(
"plugins/servlet/oauth/request-token"),
access_token_url=base_url.relative(
"plugins/servlet/oauth/access-token"),
authorization_url=base_url.relative("plugins/servlet/oauth/authorize"),
redirect_url=redirect_url,
redirect_to=redirect_to,
login_url=login_url,
authorized_url=authorized_url,
)
jira_bp.session.headers["Content-Type"] = "application/json"
@jira_bp.before_app_request
def set_applocal_session():
ctx = stack.top
ctx.jira_oauth = jira_bp.session
return jira_bp
def start_requests(self):
"""
Gets the spider started.
If both `self.login_email` and `self.login_password` are set,
this method generates a request to login with those credentials.
Otherwise, this method generates a request to go to the "auto auth"
page and get credentials from there. Either way, this method
doesn't actually generate requests from `self.start_urls` -- that is
handled by the `after_initial_login()` and `after_auto_auth()`
methods.
"""
if self.login_email and self.login_password:
login_url = (URLObject("http://").with_hostname(
self.domain).with_port(self.port).with_path(LOGIN_HTML_PATH))
yield scrapy.Request(
login_url,
callback=self.after_initial_csrf,
)
else:
self.logger.info(
"email/password unset, fetching credentials via auto_auth")
auth_url = (URLObject("http://").with_hostname(
self.domain).with_port(
self.port).with_path(AUTO_AUTH_PATH).set_query_params(
staff='true',
course_id=self.course_key,
))
# make sure to request a parseable JSON response
headers = {
b"Accept": b"application/json",
}
yield scrapy.Request(
auth_url,
headers=headers,
callback=self.after_auto_auth,
)
def __init__(self, url, db, layout, response_layout=None, **kwargs):
'''
:param url: The URL to access the FileMaker server. This should contain
any authorization credentials. If a path is not provided (e.g. no
trailing slash, like ``http://username:[email protected]``) then
the default path of ``/fmi/xml/fmresultset.xml`` will be used.
:param db: The database name to access (sets the ``-db`` parameter).
:param layout: The layout to use (sets the ``-lay`` parameter).
:param response_layout: (*Optional*) The layout to use (sets the
``-lay.response`` parameter).
'''
self.url = URLObject(url).without_auth()
self.url = self.url.with_path(self.url.path
or '/fmi/xml/fmresultset.xml')
self.auth = URLObject(url).auth
self.params = QueryDict('', mutable=True)
self.dbparams = QueryDict('', mutable=True)
self.dbparams.update({
'-db': db,
'-lay': layout,
})
if response_layout:
self.dbparams['-lay.response'] = response_layout
self.params['-max'] = '50'
def __init__(self, url, strategy_name):
self.headers = {
'User-Agent': config.get_global_setting("user_agent"),
}
self.count = 0
self.url = URLObject(url)
self.strategy_name = strategy_name
self.finish_count = 0
self._cancel = False
self.cond = threading.Condition()
self.charset = config.auto_get_charset(url)
self.vcode_len = config.get_websites_setting(self.url.hostname, "vcode_len")
self.vcode_error = config.get_strategy_setting(self.strategy_name, "vcode_error")
self.inputs_dict = config.get_strategy_inputs(self.strategy_name)
results_list = config.get_strategy_setting(self.strategy_name, "results")
self.results_select = [select for col_name, select in results_list]
self.results_colname = [col_name for col_name, select in results_list]
self.search_colname = None
self.cookies = None
self.hidden_data = None
self.required_inputs = None
self.img_url = None
self.vcode = None
self.df = None
def handle_unexpected_redirect_to_login_page(self, response):
"""
This method is called if the crawler has been unexpectedly logged out.
If that happens, and the crawler requests a page that requires a
logged-in user, the crawler will be redirected to a login page,
with the originally-requested URL as the `next` query parameter.
This method simply causes the crawler to log back in using the saved
email and password credentials. We rely on the fact that the login
page will redirect the user to the URL in the `next` query parameter
if the login is successful -- this will allow the crawl to resume
where it left off.
This is method is very much like the `get_initial_login()` method,
but the callback is `self.after_login` instead of
`self.after_initial_login`.
"""
next_url = URLObject(response.url).query_dict.get("next")
login_url = (URLObject("http://").with_hostname(self.domain).with_port(
self.port).with_path(LOGIN_API_PATH))
if next_url:
login_url = login_url.set_query_param("next", next_url)
credentials = {
"email": self.login_email,
"password": self.login_password,
}
headers = {
b"X-CSRFToken": get_csrf_token(response),
}
yield scrapy.FormRequest(
login_url,
formdata=credentials,
headers=headers,
callback=self.after_login,
)
def test_offline_select_account_and_consent(make_app):
app = make_app("foo",
"bar",
offline=True,
reprompt_consent=True,
reprompt_select_account=True)
with app.test_client() as client:
resp = client.get("/google",
base_url="https://a.b.c",
follow_redirects=False)
assert resp.status_code == 302
location = URLObject(resp.headers["Location"])
assert location.query_dict["access_type"] == "offline"
assert location.query_dict["prompt"] == "consent select_account"
def test_force_reapprove():
app = Flask(__name__)
app.secret_key = "forced"
dropbox_bp = make_dropbox_blueprint("foo", "bar", force_reapprove=True)
app.register_blueprint(dropbox_bp)
with app.test_client() as client:
resp = client.get(
"/dropbox",
base_url="https://a.b.c",
follow_redirects=False,
)
# check that there is a `force_reapprove=true` query param in the redirect URL
assert resp.status_code == 302
location = URLObject(resp.headers["Location"])
assert location.query_dict["force_reapprove"] == "true"
