def test_get_involved_cs_handles_error(github, test_client):
"""
If error happens when generating the 'get_involved_cs' page, the view
should handle it and still display most of the content. The issues
section should contain an error message with some useful links
"""
def get_issues(self, *args, **kwargs):
raise RuntimeError('Ouch!')
github.get_issues = get_issues
response = test_client.get('/zapojse/')
html = extract_issues_html(response.get_data(as_text=True))
assert response.status_code == 500
message = "DIV with the 'issues-error' class isn't present in the HTML"
assert 'issues-error' in html, message
message = "Link to alternative issues listing isn't present in the HTML"
assert 'https://github.com/pyvec/zapojse/issues' in html, message
url = '{base_url}?title={title}&body={body}'.format(
base_url='https://github.com/pyvec/python.cz/issues/new',
title=url_quote_plus('Nefunguje Zapoj se'),
body=url_quote_plus('RuntimeError: Ouch!'),
)
assert url in html, "URL for filing a bug report isn't present in the HTML"
def get_crawlers(configuration, section):
"""
parse the config section for crawlers
* does recognize (by name) known and implemented crawlers only
* a robust config reading and more freedom for users
:param configuration: RawConfigParser
:param section: string
:return: list
"""
crawlers = []
for crawler_class in Crawler.__subclasses__():
crawler_class_name = crawler_class.__name__
if not configuration.has_option(section, crawler_class_name):
continue # skip crawler if not configured
crawler_config = configuration.get(section, crawler_class_name)
if not crawler_config or crawler_config.lower() == "false":
continue # skip crawler if not configured or disabled
crawler_uris = []
# mimic old behaviours for bool values
if crawler_config.lower() == "true":
if crawler_class == Pr0gramm:
crawler_config = "static"
elif crawler_class == SoupIO:
crawler_config = "everyone"
crawler_sites = [url_quote_plus(site_stripped) for site_stripped in
[site.strip() for site in crawler_config.split(",")] # trim sites
if site_stripped] # filter stripped list for valid values
if not crawler_sites:
continue # skip crawler if no valid sites configured
logger.info("found configured Crawler: %s = %s" % (crawler_class_name, repr(crawler_sites)))
if crawler_class == Reddit:
crawler_uris = ["http://www.reddit.com/r/%s" % site for site in crawler_sites]
elif crawler_class == NineGag:
crawler_uris = ["http://9gag.com/%s" % site for site in crawler_sites]
elif crawler_class == Pr0gramm:
crawler_uris = ["http://pr0gramm.com/static/%s" % site for site in crawler_sites]
elif crawler_class == SoupIO:
crawler_uris = [("http://www.soup.io/%s" if site in ["everyone"] # public site
else "http://%s.soup.io") % site # user site
for site in crawler_sites]
elif crawler_class == Instagram:
crawler_uris = ["http://instagram.com/%s" % site for site in crawler_sites]
elif crawler_class == Fourchan:
crawler_uris = ["http://boards.4chan.org/%s/" % site for site in crawler_sites]
elif crawler_class == Giphy:
crawler_uris = ["http://api.giphy.com/v1/gifs/search?q=%s" % site for site in crawler_sites]
elif crawler_class == Bildschirmarbeiter:
crawler_uris = ["http://www.bildschirmarbeiter.com/plugs/category/%s/P120/" % site for site in crawler_sites]
crawlers += [crawler_class(crawler_uri) for crawler_uri in crawler_uris]
return crawlers
def test_get_involved_cs_handles_error(github, test_client):
"""
If error happens when generating the 'get_involved_cs' page, the view
should handle it and still display most of the content. The issues
section should contain an error message with some useful links
"""
def get_issues(self, *args, **kwargs):
raise RuntimeError('Ouch!')
github.get_issues = get_issues
response = test_client.get('/zapojse/')
html = extract_issues_html(response.get_data(as_text=True))
assert response.status_code == 500
message = "DIV with the 'issues-error' class isn't present in the HTML"
assert 'issues-error' in html, message
message = "Link to alternative issues listing isn't present in the HTML"
assert 'https://github.com/pyvec/zapojse/issues' in html, message
url = '{base_url}?title={title}&body={body}'.format(
base_url='https://github.com/pyvec/python.cz/issues/new',
title=url_quote_plus('Nefunguje Zapoj se'),
body=url_quote_plus('RuntimeError: Ouch!'),
)
assert url in html, "URL for filing a bug report isn't present in the HTML"
def get_crawlers(configuration, section):
"""
parse the config section for crawlers
* does recognize (by name) known and implemented crawlers only
* a robust config reading and more freedom for users
:param configuration: RawConfigParser
:param section: string
:return: list
"""
manager = PluginManager()
manager.setPluginPlaces([configuration.get("Sites","plugin_dir")])
manager.collectPlugins()
crawlers = []
for plugin in manager.getAllPlugins():
plug = plugin.plugin_object
plug_name = plugin.name
if not configuration.has_option(section, plug_name):
log.debug("plugin {} not configured".format(plug_name))
continue # skip crawler if not configured
crawler_config = configuration.get(section, plug_name)
log.debug("{} config is: {}".format(plug_name,crawler_config))
if crawler_config.lower() == "false":
log.debug("plugin {} disabled".format(plug_name))
continue # skip crawler if disabled
crawler_uris = []
configured_categories = [url_quote_plus(site_stripped) for site_stripped in
[site.strip() for site in crawler_config.split(",")] # trim sites
if site_stripped] # filter stripped list for valid values
# plugin factories always provide a 'default' if no category is
# configured
crawlers += plug.build(configured_categories)
log.info("configured crawlers: {}".format(crawlers))
return crawlers
def authorize():
# TODO: document and validate with swagger
#
# Validate response type now, so we can add more types later without
# requiring bwcompat for non-compliant clients
response_type = request.args.get('response_type', 'token')
if response_type != 'token':
abort(400, 'invalid response_type')
if 'redirect_uri' not in request.args:
abort(400, 'missing redirect_uri')
redirect_uri = request.args['redirect_uri']
response_mode = request.args.get('response_mode', 'fragment')
client_id = request.args.get('client_id')
if not client_id or client_id == '0':
if OAUTH_REQUIRES_CLIENT_ID:
abort(400, 'missing client_id')
client_id = db.App.force_client(role='fallback').client_id
app = db.App.objects(client_id=client_id).first()
if not app:
return abort(400, f'invalid client_id {client_id}')
if not app.validate_redirect_uri(redirect_uri):
return abort(400, f'invalid redirect_uri {redirect_uri}')
callback_params = {
'client_id': client_id,
'response_mode': response_mode,
'state': request.args.get('state'),
}
if 'nonce' in request.args:
callback_params['nonce'] = request.args['nonce']
# Some OAuth servers require exact callback URL. For these, the downstream
# redirect_uri should be in the callback params. For Slack, this prevents
# the state from being present in the callback (maybe because it is too
# large?), so place it in the redirect instead.
slack_client_redirect_uri = request.url_root.rstrip('/') + url_for(
'.slack_oauth')
serverRequiresExactCallback = False
if serverRequiresExactCallback:
callback_params['redirect_uri'] = redirect_uri
else:
slack_client_redirect_uri += '?redirect_uri=' + url_quote_plus(
redirect_uri)
state = sign_json(callback_params)
email_oauth_url = url_add_query_params(
'/oauth/send_email',
redirect_uri=redirect_uri,
state=state,
)
slack_oauth_url = url_add_query_params(
"https://slack.com/oauth/authorize",
client_id=SLACK_CLIENT_ID,
redirect_uri=slack_client_redirect_uri,
scope='identity.basic,identity.email',
state=state,
team=SLACK_TEAM_ID,
) if SLACK_CLIENT_ID else None
return render_template(
'login.html',
app=app,
email_oauth_url=email_oauth_url,
slack_oauth_url=slack_oauth_url,
)
def slack_oauth():
client_redirect_uri = request.args.get('redirect_uri')
try:
callback_params = unsign_json(request.args['state'])
except (BadSignature, KeyError):
msg = 'Upstream oauth service called back with invalid state'
logging.warning(msg)
if not client_redirect_uri:
abort(500, msg)
redirect_uri = url_add_query_params(
client_redirect_uri,
error='access_denied',
error_description=msg,
)
return redirect(redirect_uri)
client_redirect_uri = client_redirect_uri or callback_params['redirect_uri']
if 'error' in request.args:
redirect_uri = url_add_query_params(client_redirect_uri,
error=request.args['error'])
return redirect(redirect_uri)
email = None
# TODO: beef up the unit case, and make this unconditional
if 'code' in request.args:
# TODO: turn these into calls to client_redirect_uri
def check_slack_api_response(response, endpoint):
if response.status_code != 200:
msg = f"{endpoint} failed: code={response.status_code}"
logging.error(msg)
abort(500, msg)
json = response.json()
if not json.get('ok'):
msg = f"{endpoint} failed: error={json['error']}"
logging.error(msg)
abort(500, msg)
return json
code = request.args['code']
slack_client_redirect_uri = request.url_root.rstrip('/') + url_for(
'.slack_oauth')
slack_client_redirect_uri += '?redirect_uri=' + url_quote_plus(
client_redirect_uri)
response = requests.get('https://slack.com/api/oauth.access',
params=dict(
client_id=SLACK_CLIENT_ID,
client_secret=SLACK_CLIENT_SECRET,
code=code,
redirect_uri=slack_client_redirect_uri,
))
slack_access_token = check_slack_api_response(
response, "Slack oauth.access")['access_token']
response = requests.get('https://slack.com/api/users.identity',
{'token': slack_access_token})
user_info = check_slack_api_response(response,
"Slack users.identity")['user']
email = user_info['email']
token = create_access_token(
client_id=callback_params.pop('client_id'),
email=email,
provider='slack',
)
redirect_uri = implicit_grant_uri(client_redirect_uri,
access_token=token,
**callback_params)
return redirect(redirect_uri)
def urlencode_filter(s):
return url_quote_plus(str(s).encode('utf8'))
def to_coord_arg_str(self) -> str:
return self.coordinate.to_coord_arg_str(
{"url": url_quote_plus(self.url)} if self.url else {})
def generateWopiSrc(fileid):
'''Returns a valid URL-encoded WOPISrc for the given fileid'''
return url_quote_plus('%s/wopi/files/%s' % (wopi.wopiurl, fileid))
def urlencode_filter(s):
return url_quote_plus(str(s).encode('utf8'))
def get_crawlers(configuration, section):
"""
parse the config section for crawlers
* does recognize (by name) known and implemented crawlers only
* a robust config reading and more freedom for users
:param configuration: RawConfigParser
:param section: string
:return: crawler, factors
"""
crawlers = {}
factors = {}
for crawler_class in Crawler.__subclasses__():
crawler_class_name = crawler_class.__name__
if not configuration.has_option(section, crawler_class_name):
continue # skip crawler if not configured
crawler_config = configuration.get(section, crawler_class_name)
if not crawler_config or crawler_config.lower() == "false":
continue # skip crawler if not configured or disabled
crawler_uris = {}
# mimic old behaviours for bool values
if crawler_config.lower() == "true":
if crawler_class == SoupIO:
crawler_config = "everyone"
crawler_sites_and_factors = [site_stripped for site_stripped in [site.strip() for site in crawler_config.split(",")] # trim sites
if site_stripped] # filter stripped list for valid values
if not crawler_sites_and_factors:
continue # skip crawler if no valid sites configured
crawler_sites = []
factors[crawler_class_name] = {}
# Separate Site and Factor
for factorPair in crawler_sites_and_factors:
if factor_separator not in factorPair:
# No Factor configured
crawler_sites.append(url_quote_plus(factorPair))
continue
factorPair_parts = [factorPairPart.strip() for factorPairPart in factorPair.split(factor_separator)]
if not factorPair_parts or not len(factorPair_parts) == 2:
continue
site = url_quote_plus(factorPair_parts[0])
factor = float(factorPair_parts[1])
crawler_sites.append(site)
if site not in factors[crawler_class_name] and 0 < factor <= 10:
factors[crawler_class_name][site] = factor
logger.info("found configured Crawler: %s = %s Factors: %s" % (
crawler_class_name, repr(crawler_sites), repr(factors[crawler_class_name])))
if crawler_class == Reddit:
crawler_uris = {site: "https://www.reddit.com/r/%s" % site for site in crawler_sites}
elif crawler_class == NineGag:
crawler_uris = {site: "https://9gag.com/%s" % site for site in crawler_sites}
elif crawler_class == Pr0gramm:
crawler_uris = {crawler_sites[0]: "https://pr0gramm.com/api/items/get"}
elif crawler_class == SoupIO:
crawler_uris = {site: ("http://www.soup.io/%s" if site in ["everyone"] # public site
else "http://%s.soup.io") % site # user site
for site in crawler_sites}
elif crawler_class == Instagram:
crawler_uris = {site: "https://instagram.com/%s" % site for site in crawler_sites}
elif crawler_class == Fourchan:
crawler_uris = {site: "https://boards.4chan.org/%s/" % site for site in crawler_sites}
elif crawler_class == Giphy:
crawler_uris = {site: "https://api.giphy.com/v1/gifs/search?q=%s" % site for site in crawler_sites}
if crawler_class_name not in crawlers:
crawlers[crawler_class_name] = {}
crawlers[crawler_class_name] = {site: crawler_class(crawler_uris[site], site) for site in crawler_uris}
return crawlers, factors
def logout():
"Remove the auth token from the session, and redirect to redirect_uri or to the login page."
session.pop('access_token', None)
redirect_uri = request.args.get('redirect_uri', url_for('admin.login'))
return redirect('/oauth/deauthorize?redirect_uri=' +
url_quote_plus(redirect_uri))
