def build_urls():
views = {
'famfamfam/get' : famfamfam.get,
'admin/configuration' : configure.nut,
'admin/eb' : configure.eb,
'admin/eb_rec' : configure.eb_rec,
'admin/eb_fix' : configure.fix_nodes,
'admin/index' : configure.list
}
# secure the admin area
from util.decorators import require_admin
for key in views:
if key.startswith('admin'):
views[key] = require_admin(views[key])
admin_tabs = []
url_map = Map(rules())
for nut in NutSettings().nuts:
mod = __import__('hazel.nuts.%s.urls' % nut, fromlist=['hazel.nuts.%s' % nut])
pub, pub_views, admin, admin_views, tabs = mod.build_rules()
url_map.add(EndpointPrefix('nut:%s/' % nut, pub))
url_map.add(EndpointPrefix('nut:%s/' % nut,
[Submount('/admin/%s' % nut, admin)]))
admin_tabs.extend([(rule.endpoint, name) for name, rule in tabs])
views.update([(rule.endpoint, fn) for rule, fn in pub_views])
# secure admin
views.update([(rule.endpoint, require_admin(fn)) for rule, fn in admin_views])
# tell the layout engine about the enabled modules
from util.decorators import jinja_const
jinja_const('admin_tabs', admin_tabs)
return url_map, views
class InputReqApp(object):
def __init__(self):
self.url_map = Map()
self.url_map.add(Rule('/test/<path:url>', endpoint=self.direct_input_request))
self.url_map.add(Rule('/test-postreq', endpoint=self.post_fullrequest))
def direct_input_request(self, environ, url=''):
inputreq = DirectWSGIInputRequest(environ)
return inputreq.reconstruct_request(url)
def post_fullrequest(self, environ):
params = dict(parse_qsl(environ.get('QUERY_STRING', '')))
inputreq = POSTInputRequest(environ)
return inputreq.reconstruct_request(params['url'])
def __call__(self, environ, start_response):
urls = self.url_map.bind_to_environ(environ)
try:
endpoint, args = urls.match()
except HTTPException as e:
return e(environ, start_response)
result = endpoint(environ, **args)
start_response('200 OK', [('Content-Type', 'text/plain; charset=utf-8')])
return [result]
def handle_url(aConfig, environ, session):
global gTileCache
def handle_index(environ, session):
statuscode, mimetype, body = 403, 'text/plain', 'Access deny'
return statuscode, mimetype, body
user_id = None
if session and '_id' in session:
user_id = session['_id']
statuscode, mimetype, body = 200, 'text/json', '{}'
app = aConfig['gConfig']['wsgi']['application']
urllist = []
for key in aConfig['gConfig']['applications'][app]['url_mapping'].keys():
urllist.append(Rule(key, endpoint=aConfig['gConfig']['applications'][app]['url_mapping'][key]))
urlmap = Map(urllist, converters={'bool': BooleanConverter})
urls = urlmap.bind_to_environ(environ)
querydict, buf, stream = get_querydict_by_GET_POST(environ)
try:
endpoint, args = urls.match()
if endpoint == 'handle_index':
statuscode, mimetype, body = handle_index(environ, session)
else:
body = json.dumps({'result':u'access_deny'}, ensure_ascii=True, indent=4)
# except HTTPException, e:
except Exception as e:
traceback.print_exc()
body = json.dumps({'result':u'error:%s' % e.message}, ensure_ascii=True, indent=4)
# if session:
# aConfig['gSessionStore'].save(session)
# for k in hh.keys():
# headers[k] = hh[k]
return statuscode, mimetype, body
class Cup(object):
def __init__(self, with_static=True):
if with_static:
self.wsgi_app = SharedDataMiddleware(self.wsgi_app, {
'/static': os.path.join(os.path.dirname(__file__), 'static')
})
self.url_map = Map()
self.views = {}
def add_url_rule(self, url, endpt, func):
self.url_map.add(Rule(url, endpoint=endpt))
self.views[endpt] = func
def getView(self, endpoint):
return self.views[endpoint]
def route(self, url):
def decorator(func):
self.add_url_rule(url, func.__name__, func)
def decorated(*args, **kwargs):
func(*args, **kwargs)
return decorated
return decorator
def dispatch_request(self, request):
adapter = self.url_map.bind_to_environ(request.environ)
try:
endpoint, values = adapter.match()
data = self.getView(endpoint)(request, **values)
return Response(data, mimetype="text/html")
except HTTPException, e:
print "e"
return e
def application(environ, start_response):
request = Request(environ)
if os.path.exists(PATH):
with open(PATH) as yaml:
data = load(yaml)
else:
data = {}
urls = {}
for reference in data.get("references", []):
urls[reference.get("alias")] = reference.get("url")
url_map = Map([
Rule('/<alias>', endpoint='shortener'),
])
adapter = url_map.bind_to_environ(request.environ)
endpoint, values = adapter.match()
alias = values["alias"]
if alias not in urls:
raise NotFound()
response = redirect(urls[alias])
return response(environ, start_response)
def handle_url(aConfig, environ, session):
def handle_index(environ, session):
statuscode, mimetype, body = handle_static(
aConfig, environ, aConfig["gConfig"]["applications"][app]["static"]["page"]["indexpage"]
)
return statuscode, mimetype, body
statuscode, mimetype, body = 200, "text/json", "{}"
app = aConfig["gConfig"]["wsgi"]["application"]
urllist = []
for key in aConfig["gConfig"]["applications"][app]["url_mapping"].keys():
urllist.append(Rule(key, endpoint=aConfig["gConfig"]["applications"][app]["url_mapping"][key]))
urlmap = Map(urllist, converters={"bool": BooleanConverter})
urls = urlmap.bind_to_environ(environ)
try:
endpoint, args = urls.match()
if endpoint == "handle_index":
statuscode, mimetype, body = handle_index(environ, session)
else:
body = json.dumps({"result": u"access_deny"}, ensure_ascii=True, indent=4)
# except HTTPException, e:
except Exception as e:
traceback.print_exc()
body = json.dumps({"result": u"error:%s" % e.message}, ensure_ascii=True, indent=4)
return statuscode, mimetype, body
class App:
def __init__(self):
self._url_map = Map(strict_slashes=False)
def route(self, rule, **kwargs):
def decorator(func):
kwargs['endpoint'] = func
self._url_map.add(Rule(rule, **kwargs))
return func
return decorator
def _dispatch(self, request):
adapter = self._url_map.bind_to_environ(request.environ)
try:
endpoint, values = adapter.match()
return endpoint(request, **values)
except HTTPException as e:
return e
def __call__(self, env, sr):
request = AppRequest(env)
response = self._dispatch(request)
after_handlers = getattr(request, '_after_request_handlers', None)
if after_handlers:
for h in after_handlers:
response = h(response) or response
return response(env, sr)
def test_basic_building():
"""Basic URL building"""
map = Map([
Rule('/', endpoint='index'),
Rule('/foo', endpoint='foo'),
Rule('/bar/<baz>', endpoint='bar'),
Rule('/bar/<int:bazi>', endpoint='bari'),
Rule('/bar/<float:bazf>', endpoint='barf'),
Rule('/bar/<path:bazp>', endpoint='barp'),
Rule('/hehe', endpoint='blah', subdomain='blah')
])
adapter = map.bind('example.org', '/', subdomain='blah')
assert adapter.build('index', {}) == 'http://example.org/'
assert adapter.build('foo', {}) == 'http://example.org/foo'
assert adapter.build('bar', {'baz': 'blub'}) == 'http://example.org/bar/blub'
assert adapter.build('bari', {'bazi': 50}) == 'http://example.org/bar/50'
assert adapter.build('barf', {'bazf': 0.815}) == 'http://example.org/bar/0.815'
assert adapter.build('barp', {'bazp': 'la/di'}) == 'http://example.org/bar/la/di'
assert adapter.build('blah', {}) == '/hehe'
assert_raises(BuildError, lambda: adapter.build('urks'))
adapter = map.bind('example.org', '/test', subdomain='blah')
assert adapter.build('index', {}) == 'http://example.org/test/'
assert adapter.build('foo', {}) == 'http://example.org/test/foo'
assert adapter.build('bar', {'baz': 'blub'}) == 'http://example.org/test/bar/blub'
assert adapter.build('bari', {'bazi': 50}) == 'http://example.org/test/bar/50'
assert adapter.build('barf', {'bazf': 0.815}) == 'http://example.org/test/bar/0.815'
assert adapter.build('barp', {'bazp': 'la/di'}) == 'http://example.org/test/bar/la/di'
assert adapter.build('blah', {}) == '/test/hehe'
class RestRouter(object):
def __init__(self, json_encoder):
self.handlers = {}
self.map = Map()
self.encoder = json_encoder
def add_rule(self, rule, fn):
if rule.endpoint in self.handlers:
raise ValueError("Endpoint {} already present".format(rule.endpoint))
self.handlers[rule.endpoint] = fn
self.map.add(rule)
def add_resource(self, service, path):
rules = RestRules(path)
for method in RestRules.METHOD_NAMES:
if callable(getattr(service, method, None)):
self.add_rule(getattr(rules, method), getattr(service, method))
def __call__(self, request, response):
matcher = self.map.bind_to_environ(request.environ)
endpoint, args = matcher.match()
handler = self.handlers[endpoint]
result = handler(args, request)
if isinstance(result, Response):
return result
elif result is not None:
response.set_data(self.encoder.encode(result))
response.content_type = 'application/json'
return response
else:
return response
def test_external_building_with_port_bind_to_environ_wrong_servername():
"""Test external URL building with port number (map.bind_to_environ) with wrong server name raises ValueError"""
map = Map([
Rule('/', endpoint='index'),
])
environ = create_environ('/', 'http://example.org:5000/')
assert_raises(ValueError, lambda: map.bind_to_environ(environ, server_name="example.org"))
def make_url_map(self):
url_map = Map()
for provider in self._providers:
rule = provider.get_url_rule()
rule.endpoint = provider
url_map.add(rule)
return url_map
def test_request_direct_charset_bug():
map = Map([Rule(u'/öäü/')])
adapter = map.bind('localhost', '/')
try:
adapter.match(u'/öäü')
except RequestRedirect, e:
assert e.new_url == 'http://localhost/%C3%B6%C3%A4%C3%BC/'
class Dispatcher(object):
"""Dispatch requests based on a WSGI environment.
The routes are loaded from the <package>/config/routes file, and each line should be blank,
a comment, or one of the following:
<route> <page class>
<route> redirect:<url>
<route> template:<filename>
"""
def __init__(self, app):
"""Load the URL routing map and instantiate the responders."""
self.app = app
# Set up URL routing
self.map = Map()
routes_file = file(os.path.join(app.directory, 'config', 'routes'), 'r')
for line in routes_file:
# Split the line from one of the documented formats
parts = line.split()
if len(parts) == 0 or parts[0][0] == '#':
# Ignore comments and blank lines
continue
if len(parts) != 2:
raise ConfigurationError("Error in routes file: %s" % line)
path, destination = parts
if ':' in destination:
# Responder explicitly specified
responder_name, extra = destination.split(':', 1)
responder_type = responder_types.get(responder_name, None)
if responder_type is None:
raise ConfigurationError("Invalid destination '%s' in routes file" % destination)
responder = responder_type(extra)
else:
# Default to PageResponder if there's no ':' in the destination
responder = PageResponder(destination)
for p, r in responder.get_routes(path): # FIXME: Better names for p and r
rule = Rule(p, endpoint=r, methods=r.methods)
self.map.add(rule)
self.map.update()
def dispatch(self, environ):
try:
request = Request(environ)
urls = self.map.bind_to_environ(environ)
responder, args = urls.match()
with Context(self.app, environ, request, args) as context:
for hook in self.app.get_hook_functions('pre-request'):
hook(context)
context.response = responder(context)
for hook in self.app.get_hook_functions('post-request'):
context.response = hook(context) or context.response
return context.response
# HTTPExceptions are returned as the response, while any other
# exceptions are re-raised to be either caught by the in-browser debugger
# or generate a 500 response.
except HTTPException, e:
return e
def test_adapter_url_parameter_sorting():
"""Optional adapter URL parameter sorting"""
map = Map([Rule("/", endpoint="index")], sort_parameters=True, sort_key=lambda x: x[1])
adapter = map.bind("localhost", "/")
assert (
adapter.build("index", {"x": 20, "y": 10, "z": 30}, force_external=True) == "http://localhost/?y=10&x=20&z=30"
)
def test_path():
"""URL routing path converter behavior"""
map = Map(
[
Rule("/", defaults={"name": "FrontPage"}, endpoint="page"),
Rule("/Special", endpoint="special"),
Rule("/<int:year>", endpoint="year"),
Rule("/<path:name>", endpoint="page"),
Rule("/<path:name>/edit", endpoint="editpage"),
Rule("/<path:name>/silly/<path:name2>", endpoint="sillypage"),
Rule("/<path:name>/silly/<path:name2>/edit", endpoint="editsillypage"),
Rule("/Talk:<path:name>", endpoint="talk"),
Rule("/User:<username>", endpoint="user"),
Rule("/User:<username>/<path:name>", endpoint="userpage"),
Rule("/Files/<path:file>", endpoint="files"),
]
)
adapter = map.bind("example.org", "/")
assert adapter.match("/") == ("page", {"name": "FrontPage"})
assert_raises(RequestRedirect, lambda: adapter.match("/FrontPage"))
assert adapter.match("/Special") == ("special", {})
assert adapter.match("/2007") == ("year", {"year": 2007})
assert adapter.match("/Some/Page") == ("page", {"name": "Some/Page"})
assert adapter.match("/Some/Page/edit") == ("editpage", {"name": "Some/Page"})
assert adapter.match("/Foo/silly/bar") == ("sillypage", {"name": "Foo", "name2": "bar"})
assert adapter.match("/Foo/silly/bar/edit") == ("editsillypage", {"name": "Foo", "name2": "bar"})
assert adapter.match("/Talk:Foo/Bar") == ("talk", {"name": "Foo/Bar"})
assert adapter.match("/User:thomas") == ("user", {"username": "******"})
assert adapter.match("/User:thomas/projects/werkzeug") == (
"userpage",
{"username": "******", "name": "projects/werkzeug"},
)
assert adapter.match("/Files/downloads/werkzeug/0.2.zip") == ("files", {"file": "downloads/werkzeug/0.2.zip"})
def test_path():
"""URL routing path converter behavior"""
map = Map([
Rule('/', defaults={'name': 'FrontPage'}, endpoint='page'),
Rule('/Special', endpoint='special'),
Rule('/<int:year>', endpoint='year'),
Rule('/<path:name>', endpoint='page'),
Rule('/<path:name>/edit', endpoint='editpage'),
Rule('/<path:name>/silly/<path:name2>', endpoint='sillypage'),
Rule('/<path:name>/silly/<path:name2>/edit', endpoint='editsillypage'),
Rule('/Talk:<path:name>', endpoint='talk'),
Rule('/User:<username>', endpoint='user'),
Rule('/User:<username>/<path:name>', endpoint='userpage'),
Rule('/Files/<path:file>', endpoint='files'),
])
adapter = map.bind('example.org', '/')
assert adapter.match('/') == ('page', {'name':'FrontPage'})
assert_raises(RequestRedirect, lambda: adapter.match('/FrontPage'))
assert adapter.match('/Special') == ('special', {})
assert adapter.match('/2007') == ('year', {'year':2007})
assert adapter.match('/Some/Page') == ('page', {'name':'Some/Page'})
assert adapter.match('/Some/Page/edit') == ('editpage', {'name':'Some/Page'})
assert adapter.match('/Foo/silly/bar') == ('sillypage', {'name':'Foo', 'name2':'bar'})
assert adapter.match('/Foo/silly/bar/edit') == ('editsillypage', {'name':'Foo', 'name2':'bar'})
assert adapter.match('/Talk:Foo/Bar') == ('talk', {'name':'Foo/Bar'})
assert adapter.match('/User:thomas') == ('user', {'username':'******'})
assert adapter.match('/User:thomas/projects/werkzeug') == ('userpage', {'username':'******', 'name':'projects/werkzeug'})
assert adapter.match('/Files/downloads/werkzeug/0.2.zip') == ('files', {'file':'downloads/werkzeug/0.2.zip'})
def check_map(uri, url_root):
"""
return a tuple of the rule and kw.
"""
# TODO: Building the Map each time this is called seems like it could be more effiecent.
c = db.cursor()
try:
c.execute(fetch_query_string('select_route_where_dynamic.sql'))
except sqlite3.OperationalError as err:
current_app.logger.error("OperationalError: %s", err)
return (None, None)
result = c.fetchall()
if result:
(routes, col_names) = rowify(result, c.description)
#current_app.logger.debug( [x['rule'] for x in routes] )
rules = map( lambda r: Rule(r['rule'], endpoint='dynamic'), routes )
d_map = Map( rules )
map_adapter = d_map.bind(url_root)
#current_app.logger.debug(uri)
try:
(rule, rule_kw) = map_adapter.match(path_info=uri, return_rule=True)
#current_app.logger.debug(rule)
return (str(rule), rule_kw)
except HTTPException:
pass
return (None, {})
def test_adapter_url_parameter_sorting():
"""Optional adapter URL parameter sorting"""
map = Map([Rule('/', endpoint='index')], sort_parameters=True,
sort_key=lambda x: x[1])
adapter = map.bind('localhost', '/')
assert adapter.build('index', {'x': 20, 'y': 10, 'z': 30},
force_external=True) == 'http://localhost/?y=10&x=20&z=30'
def __init__(self, rules=None, default_subdomain='', charset='utf-8',
strict_slashes=True, redirect_defaults=True,
converters=None, sort_parameters=False, sort_key=None,
encoding_errors='replace', host_matching=False):
_Map.__init__(self, rules, default_subdomain, charset, strict_slashes,
redirect_defaults, converters, sort_parameters,
sort_key, encoding_errors, host_matching)
class Mapper(object):
def __init__(self, import_name):
self.import_name = import_name
self.url_map = Map()
self.url_views = {}
@property
def url_rules(self):
return self.url_map.iter_rules()
def build_endpoint(self, endpoint):
return '{0}.{1}'.format(self.import_name, endpoint)
def add_route(self, rule, endpoint, view_func=None, **options):
options['endpoint'] = endpoint
self.url_map.add(Rule(rule, **options))
if view_func:
self.url_views[endpoint] = view_func
def route(self, rule, **options):
def decorator(func):
endpoint = options.pop('endpoint', func.__name__)
endpoint = self.build_endpoint(endpoint)
self.add_route(rule, endpoint, func, **options)
return func
return decorator
def add_map(self, map, prefix='', rule_factory=Submount):
self.url_views.update(map.url_views)
self.url_map.add(
rule_factory('/%s' % prefix.rstrip('/'), map.url_rules)
)
def app(req):
url_map =Map([Rule('/',endpoint='index'),
Rule('/form',endpoint='form'),
Rule('/form/<id>', endpoint='form1')])
urls = url_map.bind_to_environ(req.environ)
return urls.dispatch(lambda e,v: route[e](req,**v))
def test_external_building_with_port():
"""Test external URL building with port number"""
map = Map([
Rule('/', endpoint='index'),
])
adapter = map.bind('example.org:5000', '/')
built_url = adapter.build('index', {}, force_external=True)
assert built_url == 'http://example.org:5000/', built_url
def test_pattern(path, pattern):
pattern = "/" + pattern.strip("/") + "/<path:extra>"
adapter = Map([Rule(pattern)]).bind("dummy.invalid")
try:
endpoint, values = adapter.match(path.strip("/"))
except NotFound:
return
return values["year"], values["month"], values["day"]
def test_request_direct_charset_bug():
map = Map([Rule(u"/öäü/")])
adapter = map.bind("localhost", "/")
try:
adapter.match(u"/öäü")
except RequestRedirect, e:
print repr(e.new_url)
assert e.new_url == "http://localhost/%C3%B6%C3%A4%C3%BC/"
def test_request_redirect_default_subdomain():
map = Map([Rule(u'/foo', defaults={'bar': 42}, subdomain='test'),
Rule(u'/foo/<int:bar>', subdomain='other')])
adapter = map.bind('localhost', '/', subdomain='other')
try:
adapter.match(u'/foo/42')
except RequestRedirect, e:
assert e.new_url == 'http://test.localhost/foo'
def test_pattern(path, pattern):
pattern = '/' + pattern.strip('/') + '/<path:extra>'
adapter = Map([Rule(pattern)]).bind('dummy.invalid')
try:
endpoint, values = adapter.match(path.strip('/'))
except NotFound:
return
return values['year'], values['month'], values['day']
def test_request_redirect_default():
map = Map([Rule(u'/foo', defaults={'bar': 42}),
Rule(u'/foo/<int:bar>')])
adapter = map.bind('localhost', '/')
try:
adapter.match(u'/foo/42')
except RequestRedirect, e:
assert e.new_url == 'http://localhost/foo'
def test_rule_templates():
"""Rule templates"""
testcase = RuleTemplate(
[ Submount('/test/$app',
[ Rule('/foo/', endpoint='handle_foo')
, Rule('/bar/', endpoint='handle_bar')
, Rule('/baz/', endpoint='handle_baz')
]),
EndpointPrefix('${app}',
[ Rule('/${app}-blah', endpoint='bar')
, Rule('/${app}-meh', endpoint='baz')
]),
Subdomain('$app',
[ Rule('/blah', endpoint='x_bar')
, Rule('/meh', endpoint='x_baz')
])
])
url_map = Map(
[ testcase(app='test1')
, testcase(app='test2')
, testcase(app='test3')
, testcase(app='test4')
])
out = sorted([(x.rule, x.subdomain, x.endpoint)
for x in url_map.iter_rules()])
assert out == ([
('/blah', 'test1', 'x_bar'),
('/blah', 'test2', 'x_bar'),
('/blah', 'test3', 'x_bar'),
('/blah', 'test4', 'x_bar'),
('/meh', 'test1', 'x_baz'),
('/meh', 'test2', 'x_baz'),
('/meh', 'test3', 'x_baz'),
('/meh', 'test4', 'x_baz'),
('/test/test1/bar/', '', 'handle_bar'),
('/test/test1/baz/', '', 'handle_baz'),
('/test/test1/foo/', '', 'handle_foo'),
('/test/test2/bar/', '', 'handle_bar'),
('/test/test2/baz/', '', 'handle_baz'),
('/test/test2/foo/', '', 'handle_foo'),
('/test/test3/bar/', '', 'handle_bar'),
('/test/test3/baz/', '', 'handle_baz'),
('/test/test3/foo/', '', 'handle_foo'),
('/test/test4/bar/', '', 'handle_bar'),
('/test/test4/baz/', '', 'handle_baz'),
('/test/test4/foo/', '', 'handle_foo'),
('/test1-blah', '', 'test1bar'),
('/test1-meh', '', 'test1baz'),
('/test2-blah', '', 'test2bar'),
('/test2-meh', '', 'test2baz'),
('/test3-blah', '', 'test3bar'),
('/test3-meh', '', 'test3baz'),
('/test4-blah', '', 'test4bar'),
('/test4-meh', '', 'test4baz')
])
class BrownAnt(object):
"""The app which could manage whole crawler system."""
def __init__(self):
self.url_map = Map(strict_slashes=False, host_matching=True)
def add_url_rule(self, host, rule_string, endpoint, **options):
"""Add a url rule to the app instance.
The url rule is the same with Flask apps and other Werkzeug apps.
:param host: the matched hostname. e.g. "www.python.org"
:param rule_string: the matched path pattern. e.g. "/news/<int:id>"
:param endpoint: the endpoint name as a dispatching key such as the
qualified name of the object.
"""
rule = Rule(rule_string, host=host, endpoint=endpoint, **options)
self.url_map.add(rule)
def parse_url(self, url_string):
"""Parse the URL string with the url map of this app instance.
:param url_string: the origin URL string.
:returns: the tuple as `(url, url_adapter, query_args)`, the url is
parsed by the standard library `urlparse`, the url_adapter is
from the werkzeug bound URL map, the query_args is a
multidict from the werkzeug.
"""
url = urlparse.urlparse(url_string)
url_adapter = self.url_map.bind(server_name=url.hostname,
url_scheme=url.scheme,
path_info=url.path)
query_args = url_decode(url.query)
return url, url_adapter, query_args
def dispatch_url(self, url_string):
"""Dispatch the URL string to the target endpoint function.
:param url_string: the origin URL string.
:returns: the return value of calling dispatched function.
"""
url, url_adapter, query_args = self.parse_url(url_string)
try:
endpoint, kwargs = url_adapter.match()
except NotFound:
raise NotSupported(url_string)
handler = import_string(endpoint)
request = Request(args=query_args)
return handler(request, **kwargs)
def mount_site(self, site):
"""Mount a supported site to this app instance.
:param site: the site instance be mounted.
"""
site.play_actions(target=self)
def server(path='/'):
try:
account = g.account
application = g.account.application
# prepare the jinja environment. This is used for regular routes
# and special handlers such as 404
template_lookup = {t.key: t.jinja2 for t in application.templates}
loader = DictLoader(template_lookup)
jinja_env = SandboxedEnvironment(
extensions=['application.app.pyjade.ext.jinja.PyJadeExtension'],
loader=loader)
# default helper utils
path = PathUtil(request.environ)
get = GetUtil(request)
static = StaticUtil()
# load template data. 404 can also use these
template_data = {}
template_data['path'] = path
template_data['get'] = get
template_data['cms'] = cms
template_data['static'] = static
template_data['deployment'] = config.TEMPLATE_GLOBAL_DEPLOYMENT
template_data['markdown'] = service.markdown
template_content = {}
for content in application.static_contents:
template_content.update(content.data)
template_data['content'] = template_content
# find the route with werkzeug
url_map = Map()
for route in application.routes:
# skip non string rules like 404. These should be handled by exceptions
if not route.rule.isnumeric():
url_map.add(Rule(route.rule, endpoint=route.template_name))
urls = url_map.bind_to_environ(request.environ)
endpoint, args = urls.match()
template_data['path'].add_placeholders(args)
app_template = jinja_env.get_template(endpoint)
page_content = app_template.render(**template_data)
app.record_transfer(page_content)
return page_content
except NotFound as e:
# find the template for a 404 handler if specified
for route in application.routes:
if route.rule == '404':
app_template = jinja_env.get_template(route.template_name)
not_found_page = app_template.render(**template_data)
app.record_transfer(not_found_page)
return not_found_page, 404
return '404', 404
except Exception as e:
return '500 internal error', 500
class Sixpack(object):
def __init__(self, redis_conn):
self.redis = redis_conn
self.statsd = init_statsd(cfg) if cfg.get('metrics') else None
self.config = cfg
self.url_map = Map([
Rule('/', endpoint='home'),
Rule('/_status', endpoint='status'),
Rule('/participate', endpoint='participate'),
Rule('/convert', endpoint='convert'),
Rule('/experiments/<name>', endpoint='experiment_details'),
Rule('/traffic', endpoint='traffic'),
Rule('/favicon.ico', endpoint='favicon'),
Rule('/alt_fractions', endpoint='alt_fractions')
])
def __call__(self, environ, start_response):
return self.wsgi_app(environ, start_response)
def wsgi_app(self, environ, start_response):
request = Request(environ)
if self.config.get('metrics'):
dispatcher = self.dispatch_request_with_metrics
else:
dispatcher = self.dispatch_request
response = dispatcher(request)
return response(environ, start_response)
def dispatch_request(self, request):
adapter = self.url_map.bind_to_environ(request.environ)
try:
endpoint, values = adapter.match()
return getattr(self, 'on_' + endpoint)(request, **values)
except NotFound:
return json_error({"message": "not found"}, request, 404)
except HTTPException:
return json_error({"message": "an internal error has occurred"},
request, 500)
def _incr_status_code(self, code):
self.statsd.incr('response_code.{}'.format(code))
def dispatch_request_with_metrics(self, request):
adapter = self.url_map.bind_to_environ(request.environ)
try:
endpoint, values = adapter.match()
with self.statsd.timer('{}.response_time'.format(endpoint)):
response = getattr(self, 'on_' + endpoint)(request, **values)
self.statsd.incr('{}.count'.format(endpoint))
self._incr_status_code(response.status_code)
return response
except NotFound:
self._incr_status_code(404)
return json_error({"message": "not found"}, request, 404)
except HTTPException:
self._incr_status_code(500)
return json_error({"message": "an internal error has occurred"},
request, 500)
@service_unavailable_on_connection_error
def on_traffic(self, request):
experiment_name = request.args.get('experiment')
traffic_fraction = request.args.get('traffic_fraction')
if traffic_fraction is None or experiment_name is None:
return json_error(
{'message': 'set traffic fail because of missing arguments'},
request, 400)
traffic_fraction = float(traffic_fraction)
try:
exp = Experiment.find(experiment_name, redis=self.redis)
exp.set_traffic_fraction(traffic_fraction)
exp.save()
except ValueError as e:
return json_error({'message': str(e)}, request, 400)
resp = {
'experiment': {
'name': experiment_name,
},
'traffic_fraction': traffic_fraction,
'status': 'ok'
}
return json_success(resp, request)
@service_unavailable_on_connection_error
def on_alt_fractions(self, request):
experiment_name = request.args.get('experiment')
if experiment_name is None:
return json_error(
{
'message':
'set alt_fractions fail because of missing experiment name'
}, request, 400)
try:
exp = Experiment.find(experiment_name, redis=self.redis)
alt_fractions = []
for alternabive in exp.alternatives:
alt_fractions.append(request.args.get(alternabive.name))
exp.set_alt_fractions(alt_fractions)
exp.save()
except ValueError as e:
return json_error({'message': str(e)}, request, 400)
resp = {
'experiment': {
'name': experiment_name,
},
'alt_fraction': exp.alt_fractions,
'status': 'ok'
}
return json_success(resp, request)
@service_unavailable_on_connection_error
def on_status(self, request):
'''
查看redis状态
:param request:
:return:
'''
self.redis.ping() # ping成功返回json_sucess,否则返回503 json_error
return json_success({'version': __version__}, request)
def on_home(self, request):
# 显示服务器主页
dales = """
,-"-.__,-"-.__,-"-..
( C> )( C> )( C> ))
/.`-_-'||`-_-'||`-_-'/
/-"-.--,-"-.--,-"-.--/|
( C> )( C> )( C> )/ |
(|`-_-',.`-_-',.`-_-'/ |
`-----++-----++----'| |
| || || |-'
| || || |
| || || |
`-_-' `-_-' `-_-'
https://github.com/seatgeek/sixpack"""
return Response(dales)
def on_favicon(self, request):
return Response()
@service_unavailable_on_connection_error
def on_convert(self, request):
if should_exclude_visitor(request):
return json_success({'excluded': 'true'}, request)
experiment_name = request.args.get('experiment')
client_id = request.args.get('client_id')
kpi = request.args.get('kpi', None)
if client_id is None or experiment_name is None:
return json_error({'message': 'missing arguments'}, request, 400)
dt = None
if request.args.get("datetime"):
dt = dateutil.parser.parse(request.args.get("datetime"))
try:
alt = convert(
experiment_name,
client_id,
kpi=kpi, #调用转化方法
datetime=dt,
redis=self.redis)
except ValueError as e:
return json_error({'message': str(e)}, request, 400)
resp = {
'alternative': {
'name': alt.name
},
'experiment': {
'name': alt.experiment.name,
},
'conversion': {
'value': None,
'kpi': kpi
},
'client_id': client_id
}
return json_success(resp, request)
@service_unavailable_on_connection_error
def on_participate(self, request):
#获取分组列表
alts = request.args.getlist('alternatives')
# alt_fractions = request.args.getlist('alt_fractions')
experiment_name = request.args.get('experiment') #实验名称
force = request.args.get('force')
#获取record_force参数，并进行to_bool判断若为[y,ture,yes],返回Ture,否则False
record_force = to_bool(request.args.get('record_force', 'false'))
client_id = request.args.get('client_id') #获取client_id
traffic_fraction = request.args.get('traffic_fraction') #获取参与比例
if traffic_fraction is not None:
traffic_fraction = float(
traffic_fraction) #若traffic_fraction存在则转为float()
prefetch = to_bool(request.args.get(
'prefetch',
'false')) #获取prefetch，并进行to_bool判断若为[y,ture,yes],返回Ture,否则Fals
if client_id is None or experiment_name is None or alts is None: #三个必要参数，否则返回400
return json_error({'message': 'missing arguments'}, request, 400)
dt = None
if request.args.get(
"datetime"): #获取datetime参数，转化为datetime.datetime()类型，没有则默认None
dt = dateutil.parser.parse(request.args.get("datetime"))
try:
if should_exclude_visitor(
request): #判断user_agent是爬虫或者IP为指定排除的ip,则拒绝参与
exp = Experiment.find(experiment_name, redis=self.redis)
if exp.winner is not None: #判断是否已经胜出
alt = exp.winner
else:
alt = exp.control #没有胜出，返回分组中的第一个赋值给alt
else:
alt = participate(
experiment_name,
alts,
client_id,
force=force,
record_force=record_force,
traffic_fraction=traffic_fraction,
# alt_fraction=alt_fractions,
prefetch=prefetch,
datetime=dt,
redis=self.redis)
except ValueError as e:
return json_error({'message': str(e)}, request, 400)
resp = {
'alternative': {
'name': alt.name
},
'experiment': {
'name': alt.experiment.name,
},
'client_id': client_id,
'status': 'ok'
}
return json_success(resp, request)
@service_unavailable_on_connection_error
def on_experiment_details(self, request, name):
exp = Experiment.find(name, redis=self.redis)
if exp is None:
return json_error({'message': 'experiment not found'}, request,
404)
return json_success(exp.objectify_by_period('day', True), request)
class Klein(object):
"""
L{Klein} is an object which is responsible for maintaining the routing
configuration of our application.
@ivar _url_map: A C{werkzeug.routing.Map} object which will be used for
routing resolution.
@ivar _endpoints: A C{dict} mapping endpoint names to handler functions.
"""
_bound_klein_instances = weakref.WeakKeyDictionary()
def __init__(self):
self._url_map = Map()
self._endpoints = {}
self._error_handlers = []
self._instance = None
def __eq__(self, other):
if isinstance(other, Klein):
return vars(self) == vars(other)
return NotImplemented
def __ne__(self, other):
result = self.__eq__(other)
if result is NotImplemented:
return result
return not result
@property
def url_map(self):
"""
Read only property exposing L{Klein._url_map}.
"""
return self._url_map
@property
def endpoints(self):
"""
Read only property exposing L{Klein._endpoints}.
"""
return self._endpoints
def execute_endpoint(self, endpoint, *args, **kwargs):
"""
Execute the named endpoint with all arguments and possibly a bound
instance.
"""
endpoint_f = self._endpoints[endpoint]
return endpoint_f(self._instance, *args, **kwargs)
def execute_error_handler(self, handler, request, failure):
"""
Execute the passed error handler, possibly with a bound instance.
"""
return handler(self._instance, request, failure)
def resource(self):
"""
Return an L{IResource} which suitably wraps this app.
@returns: An L{IResource}
"""
return KleinResource(self)
def __get__(self, instance, owner):
"""
Get an instance of L{Klein} bound to C{instance}.
"""
if instance is None:
return self
k = self._bound_klein_instances.get(instance)
if k is None:
k = self.__class__()
k._url_map = self._url_map
k._endpoints = self._endpoints
k._error_handlers = self._error_handlers
k._instance = instance
self._bound_klein_instances[instance] = k
return k
def route(self, url, *args, **kwargs):
"""
Add a new handler for C{url} passing C{args} and C{kwargs} directly to
C{werkzeug.routing.Rule}. The handler function will be passed at least
one argument an L{twisted.web.server.Request} and any keyword arguments
taken from the C{url} pattern.
::
@app.route("/")
def index(request):
return "Hello"
@param url: A werkzeug URL pattern given to C{werkzeug.routing.Rule}.
@type url: str
@param branch: A bool indiciated if a branch endpoint should
be added that allows all child path segments that don't
match some other route to be consumed. Default C{False}.
@type branch: bool
@returns: decorated handler function.
"""
segment_count = url.count('/')
if url.endswith('/'):
segment_count -= 1
def deco(f):
kwargs.setdefault('endpoint', f.__name__)
if kwargs.pop('branch', False):
branchKwargs = kwargs.copy()
branchKwargs['endpoint'] = branchKwargs['endpoint'] + '_branch'
@wraps(f)
def branch_f(instance, request, *a, **kw):
IKleinRequest(request).branch_segments = kw.pop(
'__rest__', '').split('/')
return _call(instance, f, request, *a, **kw)
branch_f.segment_count = segment_count
self._endpoints[branchKwargs['endpoint']] = branch_f
self._url_map.add(
Rule(
url.rstrip('/') + '/' + '<path:__rest__>', *args,
**branchKwargs))
@wraps(f)
def _f(instance, request, *a, **kw):
return _call(instance, f, request, *a, **kw)
_f.segment_count = segment_count
self._endpoints[kwargs['endpoint']] = _f
self._url_map.add(Rule(url, *args, **kwargs))
return f
return deco
def handle_errors(self, f_or_exception, *additional_exceptions):
"""
Register an error handler. This decorator supports two syntaxes. The
simpler of these can be used to register a handler for all C{Exception}
types::
@app.handle_errors
def error_handler(request, failure):
request.setResponseCode(500)
return 'Uh oh'
Alternately, a handler can be registered for one or more specific
C{Exception} tyes::
@app.handle_errors(EncodingError, ValidationError):
def error_handler(request, failure)
request.setResponseCode(400)
return failure.getTraceback()
The handler will be passed a L{twisted.web.server.Request} as well as a
L{twisted.python.failure.Failure} instance. Error handlers may return a
deferred, a failure or a response body.
If more than one error handler is registered, the handlers will be
executed in the order in which they are defined, until a handler is
encountered which completes successfully. If no handler completes
successfully, L{twisted.web.server.Request}'s processingFailed() method
will be called.
In addition to handling errors that occur within a route handler, error
handlers also handle any C{werkzeug.exceptions.HTTPException} which is
raised during routing. In particular, C{werkzeug.exceptions.NotFound}
will be raised if no matching route is found, so to return a custom 404
users can do the following::
@app.handle_errors(NotFound)
def error_handler(request, failure):
request.setResponseCode(404)
return 'Not found'
@param f_or_exception: An error handler function, or an C{Exception}
subclass to scope the decorated handler to.
@type f_or_exception: C{function} or C{Exception}
@param additional_exceptions Additional C{Exception} subclasses to
scope the decorated function to.
@type additional_exceptions C{list} of C{Exception}s
@returns: decorated error handler function.
"""
# Try to detect calls using the "simple" @app.handle_error syntax by
# introspecting the first argument - if it isn't a type which
# subclasses Exception we assume the simple syntax was used.
if not isinstance(f_or_exception, type) or not issubclass(
f_or_exception, Exception):
return self.handle_errors(Exception)(f_or_exception)
def deco(f):
@wraps(f)
def _f(instance, request, failure):
return _call(instance, f, request, failure)
self._error_handlers.append(
([f_or_exception] + list(additional_exceptions), _f))
return _f
return deco
def run(self, host, port, logFile=None):
"""
Run a minimal twisted.web server on the specified C{port}, bound to the
interface specified by C{host} and logging to C{logFile}.
This function will run the default reactor for your platform and so
will block the main thread of your application. It should be the last
thing your klein application does.
@param host: The hostname or IP address to bind the listening socket
to. "0.0.0.0" will allow you to listen on all interfaces, and
"127.0.0.1" will allow you to listen on just the loopback interface.
@type host: str
@param port: The TCP port to accept HTTP requests on.
@type port: int
@param logFile: The file object to log to, by default C{sys.stdout}
@type logFile: file object
"""
if logFile is None:
logFile = sys.stdout
log.startLogging(logFile)
reactor.listenTCP(port, Site(self.resource()), interface=host)
reactor.run()
class KlangbeckenAPI:
def __init__(self, stand_alone=False):
self.data_dir = os.environ.get('KLANGBECKEN_DATA',
'/var/lib/klangbecken')
self.secret = os.environ['KLANGBECKEN_API_SECRET']
self.url_map = Map()
# register the TXXX key so that we can access it later as
# mutagenfile['rg_track_gain']
EasyID3.RegisterTXXXKey(key='track_gain', desc='REPLAYGAIN_TRACK_GAIN')
EasyID3.RegisterTXXXKey(key='cue_in', desc='CUE_IN')
EasyID3.RegisterTXXXKey(key='cue_out', desc='CUE_OUT')
root_url = '/<any(' + ', '.join(PLAYLISTS) + '):category>/'
mappings = [
('/login/', ('GET', 'POST'), 'login'),
('/logout/', ('POST', ), 'logout'),
(root_url, ('GET', ), 'list'),
(root_url + '<filename>', ('GET', ), 'get'),
(root_url, ('POST', ), 'upload'),
(root_url + '<filename>', ('PUT', ), 'update'),
(root_url + '<filename>', ('DELETE', ), 'delete'),
]
if stand_alone:
# Serve html and prefix calls to api
mappings = [('/api' + path, methods, endpoint)
for path, methods, endpoint in mappings]
mappings.append(('/', ('GET', ), 'static'))
mappings.append(('/<path:path>', ('GET', ), 'static'))
cur_dir = os.path.dirname(os.path.realpath(__file__))
dist_dir = open(pjoin(cur_dir, '.dist_dir')).read().strip()
self.static_dir = pjoin(cur_dir, dist_dir)
for path, methods, endpoint in mappings:
self.url_map.add(Rule(path, methods=methods, endpoint=endpoint))
def _full_path(self, path):
return pjoin(self.data_dir, path)
def _replaygain_analysis(self, mutagenfile):
bs1770gain_cmd = [
"/usr/bin/bs1770gain", "--ebu", "--xml", mutagenfile.filename
]
output = subprocess.check_output(bs1770gain_cmd)
bs1770gain = ElementTree.fromstring(output)
# lu is in bs1770gain > album > track > integrated as an attribute
track_gain = bs1770gain.find('./album/track/integrated').attrib['lu']
mutagenfile['track_gain'] = track_gain + ' dB'
def _silan_analysis(self, mutagenfile):
silan_cmd = [
'/usr/bin/silan', '--format', 'json', mutagenfile.filename
]
output = subprocess.check_output(silan_cmd)
cue_points = json.loads(output)['sound'][0]
mutagenfile['cue_in'] = str(cue_points[0])
mutagenfile['cue_out'] = str(cue_points[1])
def __call__(self, environ, start_response):
request = Request(environ)
adapter = self.url_map.bind_to_environ(request.environ)
session = request.client_session
try:
endpoint, values = adapter.match()
if endpoint not in ['login', 'static'
] and (session.new or 'user' not in session):
raise Unauthorized()
response = getattr(self, 'on_' + endpoint)(request, **values)
except HTTPException as e:
response = e
return response(environ, start_response)
def on_login(self, request):
if request.remote_user is None:
raise Unauthorized()
response = Response(json.dumps({'status': 'OK'}), mimetype='text/json')
session = request.client_session
session['user'] = request.environ['REMOTE_USER']
session.save_cookie(response)
return response
def on_logout(self, request):
response = Response(json.dumps({'status': 'OK'}), mimetype='text/json')
session = request.client_session
del session['user']
session.save_cookie(response)
return response
def on_list(self, request, category):
cat_dir = self._full_path(category)
filenames = os.listdir(cat_dir)
tuples = [(filename, os.path.join(category, filename))
for filename in filenames]
tuples = [
(filename, path, mutagen.File(self._full_path(path), easy=True))
for (filename, path) in tuples
if os.path.isfile(self._full_path(path)) and path.endswith('.mp3')
]
counter = Counter(
path.strip()
for path in open(self._full_path(category + ".m3u")).readlines())
# FIXME: cue-points and replaygain
dicts = [{
'filename': filename,
'path': path,
'artist': mutagenfile.get('artist', [''])[0],
'title': mutagenfile.get('title', [''])[0],
'album': mutagenfile.get('album', [''])[0],
'length': float(mutagenfile.info.length),
'mtime': os.stat(self._full_path(path)).st_mtime,
'repeate': counter[path],
} for (filename, path, mutagenfile) in tuples]
data = sorted(dicts, key=lambda v: v['mtime'], reverse=True)
return Response(json.dumps(data,
indent=2,
sort_keys=True,
ensure_ascii=True),
mimetype='text/json')
def on_get(self, request, category, filename):
path = pjoin(category, secure_filename(filename))
full_path = self._full_path(path)
if not os.path.exists(full_path):
raise NotFound()
return Response(wrap_file(request.environ, open(full_path, 'rb')),
mimetype='audio/mpeg')
def on_upload(self, request, category):
file = request.files['files']
if not file:
raise UnprocessableEntity()
filename = secure_filename(file.filename)
# filename = gen_file_name(filename) # FIXME: check duplicate filenames
# mimetype = file.content_type
if not file.filename.endswith('.mp3'):
raise UnprocessableEntity('Filetype not allowed ')
# save file to disk
file_path = pjoin(category, filename)
file.save(self._full_path(file_path))
with open(self._full_path(category + '.m3u'), 'a') as f:
print(file_path, file=f)
# FIXME: silan and replaygain
# gst-launch-1.0 -t filesrc location=02_Prada.mp3 ! decodebin !
# audioconvert ! audioresample ! rganalysis ! fakesink
mutagenfile = mutagen.File(self._full_path(file_path), easy=True)
self._replaygain_analysis(mutagenfile)
self._silan_analysis(mutagenfile)
mutagenfile.save()
metadata = {
'filename': filename,
'path': file_path,
'artist': mutagenfile.get('artist', [''])[0],
'title': mutagenfile.get('title', [''])[0],
'album': mutagenfile.get('album', [''])[0],
'repeate': 1,
'length': float(mutagenfile.info.length),
'mtime': os.stat(self._full_path(file_path)).st_mtime,
}
return Response(json.dumps(metadata), mimetype='text/json')
def on_update(self, request, category, filename):
# FIXME: other values (artist, title)
path = pjoin(category, secure_filename(filename))
try:
repeates = int(json.loads(request.data)['repeate'])
except: # noqa: E722
raise UnprocessableEntity('Cannot parse PUT request')
lines = open(self._full_path(category + '.m3u')).read().split('\n')
with open(self._full_path(category + '.m3u'), 'w') as f:
for line in lines:
if line != path and line:
print(line, file=f)
for i in range(repeates):
print(path, file=f)
del i
return Response(json.dumps({'status': 'OK'}), mimetype='text/json')
def on_delete(self, request, category, filename):
path = pjoin(category, secure_filename(filename))
if not os.path.exists(self._full_path(path)):
raise NotFound()
os.remove(self._full_path(path))
lines = open(self._full_path(category + '.m3u')).read().split('\n')
with open(self._full_path(category + '.m3u'), 'w') as f:
for line in lines:
if line != path and line:
print(line, file=f)
return Response(json.dumps({'status': 'OK'}), mimetype='text/json')
def on_static(self, request, path=''):
if path in [''] + PLAYLISTS:
path = 'index.html'
path = os.path.join(self.static_dir, path)
if path.endswith('.html'):
mimetype = 'text/html'
elif path.endswith('.css'):
mimetype = 'text/css'
elif path.endswith('.js'):
mimetype = 'text/javascript'
else:
mimetype = 'text/plain'
if not os.path.isfile(path):
raise NotFound()
return Response(wrap_file(request.environ, open(path, 'rb')),
mimetype=mimetype)
class Shortly(object):
def __init__(self, config):
self.redis = redis.Redis(config["redis_host"], config["redis_port"])
template_path = os.path.join(os.path.dirname(__file__), "templates")
self.jinja_env = Environment(loader=FileSystemLoader(template_path),
autoescape=True)
self.jinja_env.filters["hostname"] = get_hostname
self.url_map = Map([
Rule("/", endpoint="home"),
Rule('/<short_id>', endpoint="follow_short_link"),
Rule('/create', endpoint="new_url"),
Rule('/<short_id>_details', endpoint="short_link_details"),
Rule('/list', endpoint='list_url')
# TODO: Добавить ендпоинты на:
# - создание шортката
# - редирект по ссылке
# - детальную информацию о ссылке
])
def render_template(self, template_name, **context):
t = self.jinja_env.get_template(template_name)
return Response(t.render(context), mimetype="text/html")
def dispatch_request(self, request):
adapter = self.url_map.bind_to_environ(request.environ)
try:
endpoint, values = adapter.match()
return getattr(self, "on_" + endpoint)(request, **values)
except NotFound:
return self.error_404()
except HTTPException as e:
return e
def wsgi_app(self, environ, start_response):
request = Request(environ)
response = self.dispatch_request(request)
return response(environ, start_response)
def on_home(self, request):
return self.render_template("homepage.html")
def on_new_url(self, request):
error = None
url = ""
if request.method == "POST":
url = request.form['url']
if not is_valid_url(url):
error = 'invalid url'
else:
id = insert_url(self.redis, url)
return redirect('/%s_details' % str(id))
# TODO: Проверить что метод для создания новой ссылки "POST"
# Проверить валидность ссылки используя is_valid_url
# Если ссылка верна - создать запись в базе и
# отправить пользователя на детальную информацию
# Если неверна - написать ошибку
return self.render_template("new_url.html", error=error, url=url)
def on_follow_short_link(self, request, short_id):
# TODO: Достать из базы запись о ссылке по ее ид (get_url)
# если такого ид в базе нет то кинуть 404 (NotFount())
# заинкрементить переход по ссылке (increment_url)
link_target = get_url(self.redis, short_id)
if not link_target:
return NotFound()
increment_url(self.redis, short_id)
return redirect(link_target)
def on_short_link_details(self, request, short_id):
# TODO: Достать из базы запись о ссылке по ее ид (get_url)
# если такого ид в базе нет то кинуть 404 (NotFount())
url = get_url(self.redis, short_id)
if not url:
return NotFound()
click_count = get_count(self.redis, short_id)
link_target = "/"
return self.render_template(
"short_link_details.html",
link_target=link_target,
short_id=short_id,
click_count=click_count,
)
def on_list_url(self, request):
# TODO: ДЗ
error = None
list_urls = get_list_urls(self.redis)
if not list_urls:
error = "no urls found"
return self.render_template("list_url.html",
error=error,
url_list=list_urls)
def error_404(self):
response = self.render_template("404.html")
response.status_code = 404
return response
def __call__(self, environ, start_response):
return self.wsgi_app(environ, start_response)
class Shortly(object):
def __init__(self):
self.redis = redis.Redis(**config.get('redis'))
template_path = os.path.join(os.path.dirname(__file__), "templates")
self.jinja_env = Environment(loader=FileSystemLoader(template_path),
autoescape=True)
self.jinja_env.filters["hostname"] = get_hostname
self.jinja_env.globals['stytle_background'] = config.get('color')
self.url_map = Map([
Rule("/", endpoint="new_url"),
Rule("/<short_id>", endpoint="follow_short_link"),
Rule("/<short_id>+", endpoint="short_link_details"),
])
def on_new_url(self, request):
error = None
url = ""
if request.method == "POST":
url = request.form["url"]
if not is_valid_url(url):
error = "Please enter a valid URL"
else:
short_id = self.insert_url(url)
return redirect("/%s+" % short_id)
return self.render_template("new_url.html", error=error, url=url)
def on_follow_short_link(self, request, short_id):
link_target = self.redis.get("url-target:" + short_id)
if link_target is None:
raise NotFound()
self.redis.incr("click-count:" + short_id)
return redirect(link_target)
def on_short_link_details(self, request, short_id):
link_target = self.redis.get("url-target:" + short_id)
if link_target is None:
raise NotFound()
click_count = int(self.redis.get("click-count:" + short_id) or 0)
return self.render_template(
"short_link_details.html",
link_target=link_target,
short_id=short_id,
click_count=click_count,
)
def error_404(self):
response = self.render_template("404.html")
response.status_code = 404
return response
def insert_url(self, url):
short_id = self.redis.get("reverse-url:" + url)
if short_id is not None:
return short_id
url_num = self.redis.incr("last-url-id")
short_id = base36_encode(url_num)
self.redis.set("url-target:" + short_id, url)
self.redis.set("reverse-url:" + url, short_id)
return short_id
def render_template(self, template_name, **context):
t = self.jinja_env.get_template(template_name)
return Response(t.render(context), mimetype="text/html")
def dispatch_request(self, request):
adapter = self.url_map.bind_to_environ(request.environ)
try:
endpoint, values = adapter.match()
return getattr(self, "on_" + endpoint)(request, **values)
except NotFound:
return self.error_404()
except HTTPException as e:
return e
def wsgi_app(self, environ, start_response):
request = Request(environ)
response = self.dispatch_request(request)
return response(environ, start_response)
def __call__(self, environ, start_response):
return self.wsgi_app(environ, start_response)
def __init__(self,
routes: RoutesConfig,
initial_types: List[type] = None) -> None:
required_type = wsgi.WSGIResponse
initial_types = initial_types or []
initial_types += [wsgi.WSGIEnviron, URLPathArgs, Exception]
rules = []
views = {}
for (path, method, view) in walk(routes):
view_signature = inspect.signature(view)
uritemplate = URITemplate(path)
# Ensure view arguments include all URL arguments
for arg in uritemplate.variable_names:
assert arg in view_signature.parameters, (
'URL argument "%s" in path "%s" must be included as a '
'keyword argument in the view function "%s"' %
(arg, path, view.__name__))
# Create a werkzeug path string
werkzeug_path = path[:]
for arg in uritemplate.variable_names:
param = view_signature.parameters[arg]
if param.annotation is inspect.Signature.empty:
converter = 'string'
elif issubclass(param.annotation, (schema.String, str)):
if getattr(param.annotation, 'format', None) == 'path':
converter = 'path'
else:
converter = 'string'
elif issubclass(param.annotation, (schema.Number, float)):
converter = 'float'
elif issubclass(param.annotation, (schema.Integer, int)):
converter = 'int'
else:
msg = 'Invalid type for path parameter, %s.' % param.annotation
raise exceptions.ConfigurationError(msg)
werkzeug_path = werkzeug_path.replace(
'{%s}' % arg, '<%s:%s>' % (converter, arg))
# Create a werkzeug routing rule
name = view.__name__
rule = Rule(werkzeug_path, methods=[method], endpoint=name)
rules.append(rule)
# Determine any inferred type annotations for the view
extra_annotations = {} # type: Dict[str, type]
for param in view_signature.parameters.values():
if param.annotation is inspect.Signature.empty:
annotated_type = str
else:
annotated_type = param.annotation
if param.name in uritemplate.variable_names:
class TypedURLPathArg(URLPathArg):
schema = annotated_type
extra_annotations[param.name] = TypedURLPathArg
elif (annotated_type in primitive_types) or issubclass(
annotated_type, schema_types):
if method in ('POST', 'PUT', 'PATCH'):
if issubclass(annotated_type, schema.Object):
class TypedDataParam(http.RequestData):
schema = annotated_type
extra_annotations[param.name] = TypedDataParam
else:
class TypedFieldParam(http.RequestField):
schema = annotated_type
extra_annotations[param.name] = TypedFieldParam
else:
class TypedQueryParam(http.QueryParam):
schema = annotated_type
extra_annotations[param.name] = TypedQueryParam
return_annotation = view_signature.return_annotation
if return_annotation is inspect.Signature.empty:
extra_annotations['return'] = http.ResponseData
elif issubclass(
return_annotation,
(schema_types, primitive_types, typing_types)): # type: ignore
extra_annotations['return'] = http.ResponseData
# Determine the pipeline for the view.
pipeline = pipelines.build_pipeline(view, initial_types,
required_type,
extra_annotations)
views[name] = Endpoint(view, pipeline)
self.exception_pipeline = pipelines.build_pipeline(
exception_handler, initial_types, required_type, {})
self.routes = routes
self.adapter = Map(rules).bind('example.com')
self.views = views
import logging
from werkzeug.routing import Map, Rule, NotFound, RequestRedirect
from werkzeug.exceptions import HTTPException
from werkzeug.wrappers import Request, Response
import dric
from future.utils import raise_from
import traceback
_logger = logging.getLogger('dric.router')
_url_map = Map()
def add(url, endpoint):
rule = Rule(url, endpoint=endpoint)
_url_map.add(rule)
def dispatch(request):
urls = _url_map.bind_to_environ(request.environ)
try:
endpoint, args = urls.match()
except HTTPException as e:
return e
args['request'] = request
_logger.debug('Incoming endpoint %s', endpoint)
try:
rep = dric.bus.publish(endpoint, **args)
except HTTPException as e:
return e
except Exception as e:
_logger.warn('Exception raised for endpoint %s: %s', endpoint, e)
200)
def index(app, request):
"""Return / -- basic stuff."""
return Response(app.render_template('main.html'),
200,
content_type='text/html')
urlmap = Map([
Rule('/', endpoint=index, methods=[
'GET',
]),
Rule('/', endpoint=create, methods=[
'POST',
]),
Rule('/<string(minlength=3):pasteid>', endpoint=show),
Rule('/<string(minlength=3):pasteid>/remove', endpoint=remove),
])
class Pastie:
SECRET_KEY = '\x85\xe1Pc\x11n\xe0\xc76\xa1\xd9\x93$\x1ei\x06'
HTTBL_KEY = 'THIS IS NOT A VALID HTTPBL KEY'
def __init__(self, data_dir='pastes/', host='http://localhost/'):
self.httpbl = HttpBL(self.HTTBL_KEY)
self.signer = Signer(self.SECRET_KEY)
class Server:
""" Main server """
def __init__(self):
self.handlers = {
"contest": ContestHandler(),
"info": InfoHandler(),
"upload": UploadHandler(),
"admin": AdminHandler(),
}
# The router tries to match the rules, the endpoint MUST be a string
# with this format
# CONTROLLER#ACTION
# Where CONTROLLER is an handler registered in self.handlers and
# ACTION is a valid
# method of that handler
self.router = Map([
Rule("/contest", methods=["GET"], endpoint="info#get_contest"),
Rule("/input/<input_id>",
methods=["GET"],
endpoint="info#get_input"),
Rule("/output/<output_id>",
methods=["GET"],
endpoint="info#get_output"),
Rule("/source/<source_id>",
methods=["GET"],
endpoint="info#get_source"),
Rule(
"/submission/<submission_id>",
methods=["GET"],
endpoint="info#get_submission",
),
Rule("/user/<token>", methods=["GET"], endpoint="info#get_user"),
Rule(
"/user/<token>/submissions/<task>",
methods=["GET"],
endpoint="info#get_submissions",
),
Rule(
"/generate_input",
methods=["POST"],
endpoint="contest#generate_input",
),
Rule("/submit", methods=["POST"], endpoint="contest#submit"),
Rule(
"/internet_detected",
methods=["POST"],
endpoint="contest#internet_detected",
),
Rule("/upload_source",
methods=["POST"],
endpoint="upload#upload_source"),
Rule("/upload_output",
methods=["POST"],
endpoint="upload#upload_output"),
Rule("/admin/upload_pack",
methods=["POST"],
endpoint="admin#upload_pack"),
Rule(
"/admin/download_results",
methods=["POST"],
endpoint="admin#download_results",
),
Rule("/admin/login", methods=["POST"], endpoint="admin#login"),
Rule("/admin/log", methods=["POST"], endpoint="admin#log"),
Rule("/admin/append_log",
methods=["POST"],
endpoint="admin#append_log"),
Rule("/admin/start", methods=["POST"], endpoint="admin#start"),
Rule(
"/admin/set_extra_time",
methods=["POST"],
endpoint="admin#set_extra_time",
),
Rule("/admin/status", methods=["POST"], endpoint="admin#status"),
Rule("/admin/pack_status",
methods=["GET"],
endpoint="admin#pack_status"),
Rule("/admin/user_list",
methods=["POST"],
endpoint="admin#user_list"),
Rule(
"/admin/drop_contest",
methods=["POST"],
endpoint="admin#drop_contest",
),
])
@responder
def __call__(self, environ, start_response):
try:
return self.wsgi_app(environ, start_response)
except:
Logger.error("UNCAUGHT_EXCEPTION", traceback.format_exc())
return InternalServerError()
def wsgi_app(self, environ, start_response):
route = self.router.bind_to_environ(environ)
request = Request(environ)
try:
endpoint, args = route.match()
except HTTPException:
Logger.warning(
"HTTP_ERROR",
"%s %s %s 404" %
(BaseHandler.get_ip(request), request.method, request.url),
)
return NotFound()
controller, action = endpoint.split("#")
return self.handlers[controller].handle(action, args, request)
def run(self):
"""
Start a greenlet with the main HTTP server loop
"""
server = gevent.pywsgi.WSGIServer((Config.address, Config.port),
self,
log=None)
try:
server.init_socket()
except OSError:
Logger.error(
"PORT_ALREADY_IN_USE",
"Address: '%s' Port: %d" % (Config.address, Config.port),
)
sys.exit(1)
greenlet = gevent.spawn(server.serve_forever)
port = "" if Config.port == 80 else ":" + str(Config.port)
Logger.info(
"SERVER_STATUS",
"Server started at http://%s%s/" % (str(Config.address), port),
)
greenlet.join()
class Isso(object):
def __init__(self, conf):
self.conf = conf
self.db = db.SQLite3(conf.get('general', 'dbpath'), conf)
self.signer = URLSafeTimedSerializer(
self.db.preferences.get("session-key"))
self.markup = html.Markup(conf.section('markup'))
self.hasher = hash.new(conf.section("hash"))
super(Isso, self).__init__(conf)
subscribers = []
for backend in conf.getlist("general", "notify"):
if backend == "stdout":
subscribers.append(Stdout(None))
elif backend in ("smtp", "SMTP"):
subscribers.append(SMTP(self))
else:
logger.warn("unknown notification backend '%s'", backend)
self.signal = ext.Signal(*subscribers)
self.urls = Map()
views.Info(self)
comments.API(self, self.hasher)
def render(self, text):
return self.markup.render(text)
def sign(self, obj):
return self.signer.dumps(obj)
def unsign(self, obj, max_age=None):
return self.signer.loads(obj,
max_age=max_age
or self.conf.getint('general', 'max-age'))
def dispatch(self, request):
local.request = request
local.host = wsgi.host(request.environ)
local.origin = origin(self.conf.getiter("general",
"host"))(request.environ)
adapter = self.urls.bind_to_environ(request.environ)
try:
handler, values = adapter.match()
except HTTPException as e:
return e
else:
try:
response = handler(request.environ, request, **values)
except HTTPException as e:
return e
except Exception:
logger.exception("%s %s", request.method,
request.environ["PATH_INFO"])
return InternalServerError()
else:
return response
def wsgi_app(self, environ, start_response):
response = self.dispatch(JSONRequest(environ))
return response(environ, start_response)
def __call__(self, environ, start_response):
return self.wsgi_app(environ, start_response)
def __init__(self, nichtparasoup: NichtParasoup):
super().__init__(nichtparasoup)
self.url_map = Map([
Rule("/", redirect_to="/index.html"),
Rule('/get', endpoint=self.on_get),
])
def create_lambda_handler(
error_handler=default_error_handler,
json_encoder=json.JSONEncoder,
application_load_balancer=False,
):
"""Create a lambda handler function with `handle` decorator as attribute
example:
lambda_handler = create_lambda_handler()
lambda_handler.handle("get")
def my_get_func(event):
pass
Inner_lambda_handler:
is the one you will receive when calling this function. It acts like a
dispatcher calling the registered http handler functions on the basis of the
incoming httpMethod.
All responses are formatted using the lambdarest.Response class.
Inner_handler:
Is the decorator function used to register funtions as handlers of
different http methods.
The inner_handler is also able to validate incoming data using a specified
JSON schema, please see http://json-schema.org for info.
"""
url_maps = Map()
def inner_lambda_handler(event, context=None):
# check if running as "aws lambda proxy"
if (not isinstance(event, dict)
or not all(key in event for key in __required_keys)
or not any(key in event for key in __either_keys)):
message = "Bad request, maybe not using Lambda Proxy?"
logging.error(message)
return Response(message, 500).to_json(
application_load_balancer=application_load_balancer)
# Save context within event for easy access
event["context"] = context
# for application load balancers, no api definition is used hence no resource is set so just use path
if "resource" not in event:
resource = event["path"]
else:
resource = event["resource"]
# Fill placeholders in resource path
if "pathParameters" in event:
resource = check_update_and_fill_resource_placeholders(
resource, event["pathParameters"])
path = resource
# Check if a path is set, if so, check if the base path is the same as
# the resource. If not, this is an api with a custom domainname.
# if so, the path will contain the actual request, but it will be
# prefixed with the basepath, which needs to be removed. Api Gateway
# only supports single level basepaths
# eg:
# path: /v2/foo/foobar
# resource: /foo/{name}
# the /v2 needs to be removed
if "path" in event and event["path"].split("/")[1] != resource.split(
"/")[1]:
path = "/%s" % "/".join(event["path"].split("/")[2:])
# proxy is a bit weird. We just replace the value in the uri with the
# actual value provided by apigw, and use that
if "{proxy+}" in resource:
path = resource.replace("{proxy+}",
event["pathParameters"]["proxy"])
method_name = event["httpMethod"].lower()
func = None
kwargs = {}
error_tuple = ("Internal server error", 500)
logging_message = "[%s][{status_code}]: {message}" % method_name
try:
# bind the mapping to an empty server name
mapping = url_maps.bind("")
rule, kwargs = mapping.match(path,
method=method_name,
return_rule=True)
func = rule.endpoint
# if this is a catch-all rule, don't send any kwargs
if rule.rule == "/<path:path>":
kwargs = {}
except NotFound as e:
logging.warning(
logging_message.format(status_code=404, message=str(e)))
error_tuple = (str(e), 404)
if func:
try:
response = func(event, **kwargs)
if not isinstance(response, Response):
# Set defaults
status_code = headers = multiValueHeaders = None
isBase64Encoded = False
if isinstance(response, tuple):
response_len = len(response)
if response_len > 3:
raise ValueError(
"Response tuple has more than 3 items")
# Unpack the tuple, missing items will be defaulted
body, status_code, headers, multiValueHeaders = response + (
None, ) * (4 - response_len)
elif isinstance(response, dict) and all(key in [
"body",
"statusCode",
"headers",
"multiValueHeaders",
"statusDescription",
"isBase64Encoded",
] for key in response.keys()):
body = response.get("body")
status_code = response.get("statusCode") or status_code
headers = response.get("headers") or headers
multiValueHeaders = (response.get("multiValueHeaders")
or multiValueHeaders)
isBase64Encoded = (response.get("isBase64Encoded")
or isBase64Encoded)
else: # if response is string, int, etc.
body = response
response = Response(body, status_code, headers,
multiValueHeaders, isBase64Encoded)
return response.to_json(
encoder=json_encoder,
application_load_balancer=application_load_balancer,
)
except ValidationError as error:
error_description = "Schema[{}] with value {}".format(
"][".join(str(error.absolute_schema_path)), error.message)
logging.warning(
logging_message.format(status_code=400,
message=error_description))
error_tuple = ("Validation Error", 400)
except ScopeMissing as error:
error_description = "Permission denied"
logging.warning(
logging_message.format(status_code=403,
message=error_description))
error_tuple = (error_description, 403)
except Exception as error:
if error_handler:
error_handler(error, method_name)
else:
raise
body, status_code = error_tuple
return Response(body, status_code).to_json(
application_load_balancer=application_load_balancer)
def inner_handler(method_name,
path="/",
schema=None,
load_json=True,
scopes=None):
if schema and not load_json:
raise ValueError(
"if schema is supplied, load_json needs to be true")
query_param_schema = None
if isinstance(schema, dict):
query_param_schema = (schema.get("properties",
{}).get("query",
{}).get("properties", {}))
def wrapper(func):
@wraps(func)
def inner(event, *args, **kwargs):
if load_json:
try:
body = json.loads(event.get("body", {}))
except:
body = {}
json_data = {
"body":
body,
"query":
__json_load_query(
event.get("queryStringParameters"),
query_param_schema=query_param_schema,
),
}
event["json"] = json_data
if schema:
# jsonschema.validate using given schema
validate(json_data, schema, **__validate_kwargs)
try:
provided_scopes = json.loads(
event["requestContext"]["authorizer"]["scopes"])
except KeyError:
provided_scopes = []
except json.decoder.JSONDecodeError:
# Ignore passed scopes if it isn't properly json encoded
provided_scopes = []
for scope in scopes or []:
if scope not in provided_scopes:
raise ScopeMissing(
"Scope: '{}' is missing".format(scope))
return func(event, *args, **kwargs)
# if this is a catch all url, make sure that it's setup correctly
if path == "*":
target_path = "/*"
else:
target_path = path
# replace the * with the werkzeug catch all path
if "*" in target_path:
target_path = target_path.replace("*", "<path:path>")
# make sure the path starts with /
if not target_path.startswith("/"):
raise ValueError("Please configure path with starting slash")
# register http handler function
rule = Rule(target_path,
endpoint=inner,
methods=[method_name.lower()])
url_maps.add(rule)
return inner
return wrapper
lambda_handler = inner_lambda_handler
lambda_handler.handle = inner_handler
return lambda_handler
def test_protocol_joining_bug():
"""Make sure the protocol joining bug is fixed"""
m = Map([Rule('/<foo>', endpoint='x')])
a = m.bind('example.org')
assert a.build('x', {'foo': 'x:y'}) == '/x:y'
assert a.build('x', {'foo': 'x:y'}, force_external=True) == 'http://example.org/x:y'
def test_adapter_match_return_rule():
"""Returning the matched Rule"""
rule = Rule('/foo/', endpoint='foo')
map = Map([rule])
adapter = map.bind('localhost', '/')
assert adapter.match('/foo/', return_rule=True) == (rule, {})
def __init__(self):
self._url_map = Map()
self._endpoints = {}
self._error_handlers = []
self._instance = None
class FrontEndApp(object):
"""Orchestrates pywb's core Wayback Machine functionality and is comprised of 2 core sub-apps and 3 optional apps.
Sub-apps:
- WarcServer: Serves the archive content (WARC/ARC and index) as well as from the live web in record/proxy mode
- RewriterApp: Rewrites the content served by pywb (if it is to be rewritten)
- WSGIProxMiddleware (Optional): If proxy mode is enabled, performs pywb's HTTP(s) proxy functionality
- AutoIndexer (Optional): If auto-indexing is enabled for the collections it is started here
- RecorderApp (Optional): Recording functionality, available when recording mode is enabled
The RewriterApp is configurable and can be set via the class var `REWRITER_APP_CLS`, defaults to RewriterApp
"""
REPLAY_API = 'http://localhost:%s/{coll}/resource/postreq'
CDX_API = 'http://localhost:%s/{coll}/index'
RECORD_SERVER = 'http://localhost:%s'
RECORD_API = 'http://localhost:%s/%s/resource/postreq?param.recorder.coll={coll}'
RECORD_ROUTE = '/record'
PROXY_CA_NAME = 'pywb HTTPS Proxy CA'
PROXY_CA_PATH = os.path.join('proxy-certs', 'pywb-ca.pem')
REWRITER_APP_CLS = RewriterApp
ALL_DIGITS = re.compile(r'^\d+$')
def __init__(self, config_file=None, custom_config=None):
"""
:param str|None config_file: Path to the config file
:param dict|None custom_config: Dictionary containing additional configuration information
"""
config_file = config_file or './config.yaml'
self.handler = self.handle_request
self.warcserver = WarcServer(config_file=config_file,
custom_config=custom_config)
self.recorder = None
self.recorder_path = None
self.proxy_default_timestamp = None
config = self.warcserver.config
self.debug = config.get('debug', False)
self.warcserver_server = GeventServer(self.warcserver, port=0)
self.proxy_prefix = None # the URL prefix to be used for the collection with proxy mode (e.g. /coll/id_/)
self.proxy_coll = None # the name of the collection that has proxy mode enabled
self.proxy_record = False # indicate if proxy recording
self.init_proxy(config)
self.init_recorder(config.get('recorder'))
self.init_autoindex(config.get('autoindex'))
static_path = config.get('static_url_path',
'pywb/static/').replace('/', os.path.sep)
self.static_handler = StaticHandler(static_path)
self.cdx_api_endpoint = config.get('cdx_api_endpoint', '/cdx')
self.query_limit = config.get('query_limit')
upstream_paths = self.get_upstream_paths(self.warcserver_server.port)
framed_replay = config.get('framed_replay', True)
self.rewriterapp = self.REWRITER_APP_CLS(framed_replay,
config=config,
paths=upstream_paths)
self.templates_dir = config.get('templates_dir', 'templates')
self.static_dir = config.get('static_dir', 'static')
metadata_templ = os.path.join(self.warcserver.root_dir, '{coll}',
'metadata.yaml')
self.metadata_cache = MetadataCache(metadata_templ)
self._init_routes()
def _init_routes(self):
"""Initialize the routes and based on the configuration file makes available
specific routes (proxy mode, record)
"""
self.url_map = Map()
self.url_map.add(
Rule('/static/_/<coll>/<path:filepath>',
endpoint=self.serve_static))
self.url_map.add(
Rule('/static/<path:filepath>', endpoint=self.serve_static))
self.url_map.add(Rule('/collinfo.json', endpoint=self.serve_listing))
if self.is_valid_coll('$root'):
coll_prefix = ''
else:
coll_prefix = '/<coll>'
self.url_map.add(Rule('/', endpoint=self.serve_home))
self._init_coll_routes(coll_prefix)
if self.proxy_prefix is not None:
# Add the proxy-fetch endpoint to enable PreservationWorker to make CORS fetches worry free in proxy mode
self.url_map.add(
Rule('/proxy-fetch/<path:url>',
endpoint=self.proxy_fetch,
methods=['GET', 'HEAD', 'OPTIONS']))
def _init_coll_routes(self, coll_prefix):
"""Initialize and register the routes for specified collection path
:param str coll_prefix: The collection path
:rtype: None
"""
routes = self._make_coll_routes(coll_prefix)
# init loc routes, if any
loc_keys = list(self.rewriterapp.loc_map.keys())
if loc_keys:
routes.append(Rule('/', endpoint=self.serve_home))
submount_route = ', '.join(loc_keys)
submount_route = '/<any({0}):lang>'.format(submount_route)
self.url_map.add(Submount(submount_route, routes))
for route in routes:
self.url_map.add(route)
def _make_coll_routes(self, coll_prefix):
"""Creates a list of standard collection routes for the
specified collection path
:param str coll_prefix: The collection path
:return: A list of route rules for the supplied collection
:rtype: list[Rule]
"""
routes = [
Rule(coll_prefix + self.cdx_api_endpoint, endpoint=self.serve_cdx),
Rule(coll_prefix + '/', endpoint=self.serve_coll_page),
Rule(coll_prefix + '/timemap/<timemap_output>/<path:url>',
endpoint=self.serve_content),
Rule(coll_prefix + '/<path:url>', endpoint=self.serve_content)
]
if self.recorder_path:
routes.append(
Rule(coll_prefix + self.RECORD_ROUTE + '/<path:url>',
endpoint=self.serve_record))
return routes
def get_upstream_paths(self, port):
"""Retrieve a dictionary containing the full URLs of the upstream apps
:param int port: The port used by the replay and cdx servers
:return: A dictionary containing the upstream paths (replay, cdx-server, record [if enabled])
:rtype: dict[str, str]
"""
base_paths = {
'replay': self.REPLAY_API % port,
'cdx-server': self.CDX_API % port,
}
if self.recorder_path:
base_paths['record'] = self.recorder_path
return base_paths
def init_recorder(self, recorder_config):
"""Initialize the recording functionality of pywb. If recording_config is None this function is a no op
:param str|dict|None recorder_config: The configuration for the recorder app
:rtype: None
"""
if not recorder_config:
self.recorder = None
self.recorder_path = None
return
if isinstance(recorder_config, str):
recorder_coll = recorder_config
recorder_config = {}
else:
recorder_coll = recorder_config['source_coll']
# cache mode
self.rec_cache_mode = recorder_config.get('cache', 'default')
dedup_policy = recorder_config.get('dedup_policy')
dedup_by_url = False
if dedup_policy == 'none':
dedup_policy = ''
if dedup_policy == 'keep':
dedup_policy = WriteDupePolicy()
elif dedup_policy == 'revisit':
dedup_policy = WriteRevisitDupePolicy()
elif dedup_policy == 'skip':
dedup_policy = SkipDupePolicy()
dedup_by_url = True
elif dedup_policy:
msg = 'Invalid option for dedup_policy: {0}'
raise Exception(msg.format(dedup_policy))
if dedup_policy:
dedup_index = WritableRedisIndexer(
redis_url=self.warcserver.dedup_index_url,
dupe_policy=dedup_policy,
rel_path_template=self.warcserver.root_dir + '/{coll}/archive')
else:
dedup_index = None
warc_writer = MultiFileWARCWriter(
self.warcserver.archive_paths,
max_size=int(recorder_config.get('rollover_size', 1000000000)),
max_idle_secs=int(recorder_config.get('rollover_idle_secs', 600)),
filename_template=recorder_config.get('filename_template'),
dedup_index=dedup_index,
dedup_by_url=dedup_by_url)
self.recorder = RecorderApp(
self.RECORD_SERVER % str(self.warcserver_server.port),
warc_writer,
accept_colls=recorder_config.get('source_filter'))
recorder_server = GeventServer(self.recorder, port=0)
self.recorder_path = self.RECORD_API % (recorder_server.port,
recorder_coll)
def init_autoindex(self, auto_interval):
"""Initialize and start the auto-indexing of the collections. If auto_interval is None this is a no op.
:param str|int auto_interval: The auto-indexing interval from the configuration file or CLI argument
"""
if not auto_interval:
return
from pywb.manager.autoindex import AutoIndexer
colls_dir = self.warcserver.root_dir if self.warcserver.root_dir else None
indexer = AutoIndexer(colls_dir=colls_dir, interval=int(auto_interval))
if not os.path.isdir(indexer.root_path):
msg = 'No managed directory "{0}" for auto-indexing'
logging.error(msg.format(indexer.root_path))
import sys
sys.exit(2)
msg = 'Auto-Indexing Enabled on "{0}", checking every {1} secs'
logging.info(msg.format(indexer.root_path, auto_interval))
indexer.start()
def is_proxy_enabled(self, environ):
"""Returns T/F indicating if proxy mode is enabled
:param dict environ: The WSGI environment dictionary for the request
:return: T/F indicating if proxy mode is enabled
:rtype: bool
"""
return self.proxy_prefix is not None and 'wsgiprox.proxy_host' in environ
def serve_home(self, environ):
"""Serves the home (/) view of pywb (not a collections)
:param dict environ: The WSGI environment dictionary for the request
:return: The WbResponse for serving the home (/) path
:rtype: WbResponse
"""
home_view = BaseInsertView(self.rewriterapp.jinja_env, 'index.html')
fixed_routes = self.warcserver.list_fixed_routes()
dynamic_routes = self.warcserver.list_dynamic_routes()
routes = fixed_routes + dynamic_routes
all_metadata = self.metadata_cache.get_all(dynamic_routes)
content = home_view.render_to_string(environ,
routes=routes,
all_metadata=all_metadata)
return WbResponse.text_response(
content, content_type='text/html; charset="utf-8"')
def serve_static(self, environ, coll='', filepath=''):
"""Serve a static file associated with a specific collection or one of pywb's own static assets
:param dict environ: The WSGI environment dictionary for the request
:param str coll: The collection the static file is associated with
:param str filepath: The file path (relative to the collection) for the static assest
:return: The WbResponse for the static asset
:rtype: WbResponse
"""
proxy_enabled = self.is_proxy_enabled(environ)
if proxy_enabled and environ.get('REQUEST_METHOD') == 'OPTIONS':
return WbResponse.options_response(environ)
if coll:
path = os.path.join(self.warcserver.root_dir, coll,
self.static_dir)
else:
path = self.static_dir
environ['pywb.static_dir'] = path
try:
response = self.static_handler(environ, filepath)
if proxy_enabled:
response.add_access_control_headers(env=environ)
return response
except Exception:
self.raise_not_found(environ, 'static_file_not_found', filepath)
def get_coll_config(self, coll):
"""Retrieve the collection config, including metadata, associated with a collection
:param str coll: The name of the collection to receive config info for
:return: The collections config
:rtype: dict
"""
coll_config = {'coll': coll, 'type': 'replay'}
if coll in self.warcserver.list_fixed_routes():
coll_config.update(self.warcserver.get_coll_config(coll))
else:
coll_config['metadata'] = self.metadata_cache.load(coll) or {}
return coll_config
def serve_coll_page(self, environ, coll='$root'):
"""Render and serve a collections search page (search.html).
:param dict environ: The WSGI environment dictionary for the request
:param str coll: The name of the collection to serve the collections search page for
:return: The WbResponse containing the collections search page
:rtype: WbResponse
"""
if not self.is_valid_coll(coll):
self.raise_not_found(environ, 'coll_not_found', coll)
self.setup_paths(environ, coll)
coll_config = self.get_coll_config(coll)
metadata = coll_config.get('metadata')
view = BaseInsertView(self.rewriterapp.jinja_env, 'search.html')
wb_prefix = environ.get('SCRIPT_NAME', '')
if wb_prefix:
wb_prefix += '/'
content = view.render_to_string(environ,
wb_prefix=wb_prefix,
coll=coll,
coll_config=coll_config,
metadata=metadata)
return WbResponse.text_response(
content, content_type='text/html; charset="utf-8"')
def serve_cdx(self, environ, coll='$root'):
"""Make the upstream CDX query for a collection and response with the results of the query
:param dict environ: The WSGI environment dictionary for the request
:param str coll: The name of the collection this CDX query is for
:return: The WbResponse containing the results of the CDX query
:rtype: WbResponse
"""
base_url = self.rewriterapp.paths['cdx-server']
# if coll == self.all_coll:
# coll = '*'
cdx_url = base_url.format(coll=coll)
if environ.get('QUERY_STRING'):
cdx_url += '&' if '?' in cdx_url else '?'
cdx_url += environ.get('QUERY_STRING')
if self.query_limit:
cdx_url += '&' if '?' in cdx_url else '?'
cdx_url += 'limit=' + str(self.query_limit)
try:
res = requests.get(cdx_url, stream=True)
content_type = res.headers.get('Content-Type')
return WbResponse.bin_stream(StreamIter(res.raw),
content_type=content_type)
except Exception as e:
return WbResponse.text_response('Error: ' + str(e),
status='400 Bad Request')
def serve_record(self, environ, coll='$root', url=''):
"""Serve a URL's content from a WARC/ARC record in replay mode or from the live web in
live, proxy, and record mode.
:param dict environ: The WSGI environment dictionary for the request
:param str coll: The name of the collection the record is to be served from
:param str url: The URL for the corresponding record to be served if it exists
:return: WbResponse containing the contents of the record/URL
:rtype: WbResponse
"""
if coll in self.warcserver.list_fixed_routes():
return WbResponse.text_response(
'Error: Can Not Record Into Custom Collection "{0}"'.format(
coll))
return self.serve_content(environ, coll, url, record=True)
def serve_content(self,
environ,
coll='$root',
url='',
timemap_output='',
record=False):
"""Serve the contents of a URL/Record rewriting the contents of the response when applicable.
:param dict environ: The WSGI environment dictionary for the request
:param str coll: The name of the collection the record is to be served from
:param str url: The URL for the corresponding record to be served if it exists
:param str timemap_output: The contents of the timemap included in the link header of the response
:param bool record: Should the content being served by recorded (save to a warc). Only valid in record mode
:return: WbResponse containing the contents of the record/URL
:rtype: WbResponse
"""
if not self.is_valid_coll(coll):
self.raise_not_found(environ, 'coll_not_found', coll)
self.setup_paths(environ, coll, record)
request_uri = environ.get('REQUEST_URI')
script_name = environ.get('SCRIPT_NAME', '') + '/'
if request_uri and request_uri.startswith(script_name):
wb_url_str = request_uri[len(script_name):]
else:
wb_url_str = to_native_str(url)
if environ.get('QUERY_STRING'):
wb_url_str += '?' + environ.get('QUERY_STRING')
coll_config = self.get_coll_config(coll)
if record:
coll_config['type'] = 'record'
coll_config['cache'] = self.rec_cache_mode
if timemap_output:
coll_config['output'] = timemap_output
# ensure that the timemap path information is not included
wb_url_str = wb_url_str.replace(
'timemap/{0}/'.format(timemap_output), '')
return self.rewriterapp.render_content(wb_url_str, coll_config,
environ)
def setup_paths(self, environ, coll, record=False):
"""Populates the WSGI environment dictionary with the path information necessary to perform a response for
content or record.
:param dict environ: The WSGI environment dictionary for the request
:param str coll: The name of the collection the record is to be served from
:param bool record: Should the content being served by recorded (save to a warc). Only valid in record mode
"""
if not coll or not self.warcserver.root_dir:
return
if coll != '$root':
pop_path_info(environ)
if record:
pop_path_info(environ)
paths = [self.warcserver.root_dir]
if coll != '$root':
paths.append(coll)
paths.append(self.templates_dir)
# jinja2 template paths always use '/' as separator
environ['pywb.templates_dir'] = '/'.join(paths)
def serve_listing(self, environ):
"""Serves the response for WARCServer fixed and dynamic listing (paths)
:param dict environ: The WSGI environment dictionary for the request
:return: WbResponse containing the frontend apps WARCServer URL paths
:rtype: WbResponse
"""
result = {
'fixed': self.warcserver.list_fixed_routes(),
'dynamic': self.warcserver.list_dynamic_routes()
}
return WbResponse.json_response(result)
def is_valid_coll(self, coll):
"""Determines if the collection name for a request is valid (exists)
:param str coll: The name of the collection to check
:return: True if the collection is valid, false otherwise
:rtype: bool
"""
# if coll == self.all_coll:
# return True
return (coll in self.warcserver.list_fixed_routes()
or coll in self.warcserver.list_dynamic_routes())
def raise_not_found(self, environ, err_type, url):
"""Utility function for raising a werkzeug.exceptions.NotFound execption with the supplied WSGI environment
and message.
:param dict environ: The WSGI environment dictionary for the request
:param str err_type: The identifier for type of error that occured
:param str url: The url of the archived page that was requested
"""
raise AppPageNotFound(err_type, url)
def _check_refer_redirect(self, environ):
"""Returns a WbResponse for a HTTP 307 redirection if the HTTP referer header is the same as the HTTP host header
:param dict environ: The WSGI environment dictionary for the request
:return: WbResponse HTTP 307 redirection
:rtype: WbResponse
"""
referer = environ.get('HTTP_REFERER')
if not referer:
return
host = environ.get('HTTP_HOST')
if host not in referer:
return
inx = referer[1:].find('http')
if not inx:
inx = referer[1:].find('///')
if inx < 0:
return
url = referer[inx + 1:]
host = referer[:inx + 1]
orig_url = environ['PATH_INFO']
if environ.get('QUERY_STRING'):
orig_url += '?' + environ['QUERY_STRING']
full_url = host + urljoin(url, orig_url)
return WbResponse.redir_response(full_url, '307 Redirect')
def __call__(self, environ, start_response):
"""Handles a request
:param dict environ: The WSGI environment dictionary for the request
:param start_response:
:return: The WbResponse for the request
:rtype: WbResponse
"""
return self.handler(environ, start_response)
def handle_request(self, environ, start_response):
"""Retrieves the route handler and calls the handler returning its the response
:param dict environ: The WSGI environment dictionary for the request
:param start_response:
:return: The WbResponse for the request
:rtype: WbResponse
"""
urls = self.url_map.bind_to_environ(environ)
try:
endpoint, args = urls.match()
# store original script_name (original prefix) before modifications are made
environ['pywb.app_prefix'] = environ.get('SCRIPT_NAME', '')
# store original script_name (original prefix) before modifications are made
environ['ORIG_SCRIPT_NAME'] = environ.get('SCRIPT_NAME')
lang = args.pop('lang', '')
if lang:
pop_path_info(environ)
environ['pywb_lang'] = lang
response = endpoint(environ, **args)
except RequestRedirect as rr:
# if werkzeug throws this, likely a missing slash redirect
# also check referrer here to avoid another redirect later
redir = self._check_refer_redirect(environ)
if redir:
return redir(environ, start_response)
response = WbResponse.redir_response(rr.new_url, '307 Redirect')
except WbException as wbe:
if wbe.status_code == 404:
redir = self._check_refer_redirect(environ)
if redir:
return redir(environ, start_response)
response = self.rewriterapp.handle_error(environ, wbe)
except Exception as e:
if self.debug:
traceback.print_exc()
response = self.rewriterapp._error_response(
environ, WbException('Internal Error: ' + str(e)))
return response(environ, start_response)
@classmethod
def create_app(cls, port):
"""Create a new instance of FrontEndApp that listens on port with a hostname of 0.0.0.0
:param int port: The port FrontEndApp is to listen on
:return: A new instance of FrontEndApp wrapped in GeventServer
:rtype: GeventServer
"""
app = FrontEndApp()
app_server = GeventServer(app, port=port, hostname='0.0.0.0')
return app_server
def init_proxy(self, config):
"""Initialize and start proxy mode. If proxy configuration entry is not contained in the config
this is a no op. Causes handler to become an instance of WSGIProxMiddleware.
:param dict config: The configuration object used to configure this instance of FrontEndApp
"""
proxy_config = config.get('proxy')
if not proxy_config:
return
if isinstance(proxy_config, str):
proxy_coll = proxy_config
proxy_config = {}
else:
proxy_coll = proxy_config['coll']
if '/' in proxy_coll:
raise Exception('Proxy collection can not contain "/"')
proxy_config['ca_name'] = proxy_config.get('ca_name',
self.PROXY_CA_NAME)
proxy_config['ca_file_cache'] = proxy_config.get(
'ca_file_cache', self.PROXY_CA_PATH)
if proxy_config.get('recording'):
logging.info(
'Proxy recording into collection "{0}"'.format(proxy_coll))
if proxy_coll in self.warcserver.list_fixed_routes():
raise Exception('Can not record into fixed collection')
proxy_route = proxy_coll + self.RECORD_ROUTE
if not config.get('recorder'):
config['recorder'] = 'live'
self.proxy_record = True
else:
logging.info(
'Proxy enabled for collection "{0}"'.format(proxy_coll))
self.proxy_record = False
proxy_route = proxy_coll
if proxy_config.get('enable_content_rewrite', True):
self.proxy_prefix = '/{0}/bn_/'.format(proxy_route)
else:
self.proxy_prefix = '/{0}/id_/'.format(proxy_route)
self.proxy_default_timestamp = proxy_config.get('default_timestamp')
if self.proxy_default_timestamp:
if not self.ALL_DIGITS.match(self.proxy_default_timestamp):
try:
self.proxy_default_timestamp = iso_date_to_timestamp(
self.proxy_default_timestamp)
except Exception:
raise Exception(
'Invalid Proxy Timestamp: Must Be All-Digit Timestamp or ISO Date Format'
)
self.proxy_coll = proxy_coll
self.handler = WSGIProxMiddleware(self.handle_request,
self.proxy_route_request,
proxy_host=proxy_config.get(
'host', 'pywb.proxy'),
proxy_options=proxy_config)
def proxy_route_request(self, url, environ):
""" Return the full url that this proxy request will be routed to
The 'environ' PATH_INFO and REQUEST_URI will be modified based on the returned url
Default is to use the 'proxy_prefix' to point to the proxy collection
"""
if self.proxy_default_timestamp:
environ[
'pywb_proxy_default_timestamp'] = self.proxy_default_timestamp
return self.proxy_prefix + url
def proxy_fetch(self, env, url):
"""Proxy mode only endpoint that handles OPTIONS requests and COR fetches for Preservation Worker.
Due to normal cross-origin browser restrictions in proxy mode, auto fetch worker cannot access the CSS rules
of cross-origin style sheets and must re-fetch them in a manner that is CORS safe. This endpoint facilitates
that by fetching the stylesheets for the auto fetch worker and then responds with its contents
:param dict env: The WSGI environment dictionary
:param str url: The URL of the resource to be fetched
:return: WbResponse that is either response to an Options request or the results of fetching url
:rtype: WbResponse
"""
if not self.is_proxy_enabled(env):
# we are not in proxy mode so just respond with forbidden
return WbResponse.text_response(
'proxy mode must be enabled to use this endpoint',
status='403 Forbidden')
if env.get('REQUEST_METHOD') == 'OPTIONS':
return WbResponse.options_response(env)
# ensure full URL
url = env['REQUEST_URI'].split('/proxy-fetch/', 1)[-1]
env['REQUEST_URI'] = self.proxy_prefix + url
env['PATH_INFO'] = self.proxy_prefix + env['PATH_INFO'].split(
'/proxy-fetch/', 1)[-1]
# make request using normal serve_content
response = self.serve_content(env,
self.proxy_coll,
url,
record=self.proxy_record)
# for WR
if isinstance(response, WbResponse):
response.add_access_control_headers(env=env)
return response
def __init__(self):
self.handlers = {
"contest": ContestHandler(),
"info": InfoHandler(),
"upload": UploadHandler(),
"admin": AdminHandler(),
}
# The router tries to match the rules, the endpoint MUST be a string
# with this format
# CONTROLLER#ACTION
# Where CONTROLLER is an handler registered in self.handlers and
# ACTION is a valid
# method of that handler
self.router = Map([
Rule("/contest", methods=["GET"], endpoint="info#get_contest"),
Rule("/input/<input_id>",
methods=["GET"],
endpoint="info#get_input"),
Rule("/output/<output_id>",
methods=["GET"],
endpoint="info#get_output"),
Rule("/source/<source_id>",
methods=["GET"],
endpoint="info#get_source"),
Rule(
"/submission/<submission_id>",
methods=["GET"],
endpoint="info#get_submission",
),
Rule("/user/<token>", methods=["GET"], endpoint="info#get_user"),
Rule(
"/user/<token>/submissions/<task>",
methods=["GET"],
endpoint="info#get_submissions",
),
Rule(
"/generate_input",
methods=["POST"],
endpoint="contest#generate_input",
),
Rule("/submit", methods=["POST"], endpoint="contest#submit"),
Rule(
"/internet_detected",
methods=["POST"],
endpoint="contest#internet_detected",
),
Rule("/upload_source",
methods=["POST"],
endpoint="upload#upload_source"),
Rule("/upload_output",
methods=["POST"],
endpoint="upload#upload_output"),
Rule("/admin/upload_pack",
methods=["POST"],
endpoint="admin#upload_pack"),
Rule(
"/admin/download_results",
methods=["POST"],
endpoint="admin#download_results",
),
Rule("/admin/login", methods=["POST"], endpoint="admin#login"),
Rule("/admin/log", methods=["POST"], endpoint="admin#log"),
Rule("/admin/append_log",
methods=["POST"],
endpoint="admin#append_log"),
Rule("/admin/start", methods=["POST"], endpoint="admin#start"),
Rule(
"/admin/set_extra_time",
methods=["POST"],
endpoint="admin#set_extra_time",
),
Rule("/admin/status", methods=["POST"], endpoint="admin#status"),
Rule("/admin/pack_status",
methods=["GET"],
endpoint="admin#pack_status"),
Rule("/admin/user_list",
methods=["POST"],
endpoint="admin#user_list"),
Rule(
"/admin/drop_contest",
methods=["POST"],
endpoint="admin#drop_contest",
),
])
class Flask(_PackageBoundObject):
"""The flask object implements a WSGI application and acts as the central
object. It is passed the name of the module or package of the
application. Once it is created it will act as a central registry for
the view functions, the URL rules, template configuration and much more.
The name of the package is used to resolve resources from inside the
package or the folder the module is contained in depending on if the
package parameter resolves to an actual python package (a folder with
an `__init__.py` file inside) or a standard module (just a `.py` file).
For more information about resource loading, see :func:`open_resource`.
Usually you create a :class:`Flask` instance in your main module or
in the `__init__.py` file of your package like this::
from flask import Flask
app = Flask(__name__)
"""
#: the class that is used for request objects. See :class:`~flask.request`
#: for more information.
request_class = Request
#: the class that is used for response objects. See
#: :class:`~flask.Response` for more information.
response_class = Response
#: path for the static files. If you don't want to use static files
#: you can set this value to `None` in which case no URL rule is added
#: and the development server will no longer serve any static files.
static_path = '/static'
#: if a secret key is set, cryptographic components can use this to
#: sign cookies and other things. Set this to a complex random value
#: when you want to use the secure cookie for instance.
secret_key = None
#: The secure cookie uses this for the name of the session cookie
session_cookie_name = 'session'
#: A :class:`~datetime.timedelta` which is used to set the expiration
#: date of a permanent session. The default is 31 days which makes a
#: permanent session survive for roughly one month.
permanent_session_lifetime = timedelta(days=31)
#: options that are passed directly to the Jinja2 environment
jinja_options = ImmutableDict(
autoescape=True,
extensions=['jinja2.ext.autoescape', 'jinja2.ext.with_']
)
def __init__(self, import_name):
_PackageBoundObject.__init__(self, import_name)
#: the debug flag. Set this to `True` to enable debugging of
#: the application. In debug mode the debugger will kick in
#: when an unhandled exception ocurrs and the integrated server
#: will automatically reload the application if changes in the
#: code are detected.
self.debug = False
#: a dictionary of all view functions registered. The keys will
#: be function names which are also used to generate URLs and
#: the values are the function objects themselves.
#: to register a view function, use the :meth:`route` decorator.
self.view_functions = {}
#: a dictionary of all registered error handlers. The key is
#: be the error code as integer, the value the function that
#: should handle that error.
#: To register a error handler, use the :meth:`errorhandler`
#: decorator.
self.error_handlers = {}
#: a dictionary with lists of functions that should be called at the
#: beginning of the request. The key of the dictionary is the name of
#: the module this function is active for, `None` for all requests.
#: This can for example be used to open database connections or
#: getting hold of the currently logged in user. To register a
#: function here, use the :meth:`before_request` decorator.
self.before_request_funcs = {}
#: a dictionary with lists of functions that should be called after
#: each request. The key of the dictionary is the name of the module
#: this function is active for, `None` for all requests. This can for
#: example be used to open database connections or getting hold of the
#: currently logged in user. To register a function here, use the
#: :meth:`before_request` decorator.
self.after_request_funcs = {}
#: a dictionary with list of functions that are called without arguments
#: to populate the template context. They key of the dictionary is the
#: name of the module this function is active for, `None` for all
#: requests. Each returns a dictionary that the template context is
#: updated with. To register a function here, use the
#: :meth:`context_processor` decorator.
self.template_context_processors = {
None: [_default_template_ctx_processor]
}
#: the :class:`~werkzeug.routing.Map` for this instance. You can use
#: this to change the routing converters after the class was created
#: but before any routes are connected. Example::
#:
#: from werkzeug import BaseConverter
#:
#: class ListConverter(BaseConverter):
#: def to_python(self, value):
#: return value.split(',')
#: def to_url(self, values):
#: return ','.join(BaseConverter.to_url(value)
#: for value in values)
#:
#: app = Flask(__name__)
#: app.url_map.converters['list'] = ListConverter
self.url_map = Map()
if self.static_path is not None:
self.add_url_rule(self.static_path + '/<filename>',
build_only=True, endpoint='static')
if pkg_resources is not None:
target = (self.import_name, 'static')
else:
target = os.path.join(self.root_path, 'static')
self.wsgi_app = SharedDataMiddleware(self.wsgi_app, {
self.static_path: target
})
#: the Jinja2 environment. It is created from the
#: :attr:`jinja_options` and the loader that is returned
#: by the :meth:`create_jinja_loader` function.
self.jinja_env = Environment(loader=self.create_jinja_loader(),
**self.jinja_options)
self.jinja_env.globals.update(
url_for=url_for,
get_flashed_messages=get_flashed_messages
)
self.jinja_env.filters['tojson'] = _tojson_filter
def create_jinja_loader(self):
"""Creates the Jinja loader. By default just a package loader for
the configured package is returned that looks up templates in the
`templates` folder. To add other loaders it's possible to
override this method.
"""
if pkg_resources is None:
return FileSystemLoader(os.path.join(self.root_path, 'templates'))
return PackageLoader(self.import_name)
def update_template_context(self, context):
"""Update the template context with some commonly used variables.
This injects request, session and g into the template context.
:param context: the context as a dictionary that is updated in place
to add extra variables.
"""
funcs = self.template_context_processors[None]
mod = _request_ctx_stack.top.request.module
if mod is not None and mod in self.template_context_processors:
funcs = chain(funcs, self.template_context_processors[mod])
for func in funcs:
context.update(func())
def run(self, host='127.0.0.1', port=5000, **options):
"""Runs the application on a local development server. If the
:attr:`debug` flag is set the server will automatically reload
for code changes and show a debugger in case an exception happened.
:param host: the hostname to listen on. set this to ``'0.0.0.0'``
to have the server available externally as well.
:param port: the port of the webserver
:param options: the options to be forwarded to the underlying
Werkzeug server. See :func:`werkzeug.run_simple`
for more information.
"""
from werkzeug import run_simple
if 'debug' in options:
self.debug = options.pop('debug')
options.setdefault('use_reloader', self.debug)
options.setdefault('use_debugger', self.debug)
return run_simple(host, port, self, **options)
def test_client(self):
"""Creates a test client for this application. For information
about unit testing head over to :ref:`testing`.
"""
from werkzeug import Client
return Client(self, self.response_class, use_cookies=True)
def open_session(self, request):
"""Creates or opens a new session. Default implementation stores all
session data in a signed cookie. This requires that the
:attr:`secret_key` is set.
:param request: an instance of :attr:`request_class`.
"""
key = self.secret_key
if key is not None:
return Session.load_cookie(request, self.session_cookie_name,
secret_key=key)
def save_session(self, session, response):
"""Saves the session if it needs updates. For the default
implementation, check :meth:`open_session`.
:param session: the session to be saved (a
:class:`~werkzeug.contrib.securecookie.SecureCookie`
object)
:param response: an instance of :attr:`response_class`
"""
expires = None
if session.permanent:
expires = datetime.utcnow() + self.permanent_session_lifetime
session.save_cookie(response, self.session_cookie_name,
expires=expires, httponly=True)
def register_module(self, module, **options):
"""Registers a module with this application. The keyword argument
of this function are the same as the ones for the constructor of the
:class:`Module` class and will override the values of the module if
provided.
"""
options.setdefault('url_prefix', module.url_prefix)
state = _ModuleSetupState(self, **options)
for func in module._register_events:
func(state)
def add_url_rule(self, rule, endpoint=None, view_func=None, **options):
"""Connects a URL rule. Works exactly like the :meth:`route`
decorator. If a view_func is provided it will be registered with the
endpoint.
Basically this example::
@app.route('/')
def index():
pass
Is equivalent to the following::
def index():
pass
app.add_url_rule('/', 'index', index)
If the view_func is not provided you will need to connect the endpoint
to a view function like so::
app.view_functions['index'] = index
.. versionchanged:: 0.2
`view_func` parameter added.
:param rule: the URL rule as string
:param endpoint: the endpoint for the registered URL rule. Flask
itself assumes the name of the view function as
endpoint
:param view_func: the function to call when serving a request to the
provided endpoint
:param options: the options to be forwarded to the underlying
:class:`~werkzeug.routing.Rule` object
"""
if endpoint is None:
assert view_func is not None, 'expected view func if endpoint ' \
'is not provided.'
endpoint = view_func.__name__
options['endpoint'] = endpoint
options.setdefault('methods', ('GET',))
self.url_map.add(Rule(rule, **options))
if view_func is not None:
self.view_functions[endpoint] = view_func
def route(self, rule, **options):
"""A decorator that is used to register a view function for a
given URL rule. Example::
@app.route('/')
def index():
return 'Hello World'
Variables parts in the route can be specified with angular
brackets (``/user/<username>``). By default a variable part
in the URL accepts any string without a slash however a different
converter can be specified as well by using ``<converter:name>``.
Variable parts are passed to the view function as keyword
arguments.
The following converters are possible:
=========== ===========================================
`int` accepts integers
`float` like `int` but for floating point values
`path` like the default but also accepts slashes
=========== ===========================================
Here some examples::
@app.route('/')
def index():
pass
@app.route('/<username>')
def show_user(username):
pass
@app.route('/post/<int:post_id>')
def show_post(post_id):
pass
An important detail to keep in mind is how Flask deals with trailing
slashes. The idea is to keep each URL unique so the following rules
apply:
1. If a rule ends with a slash and is requested without a slash
by the user, the user is automatically redirected to the same
page with a trailing slash attached.
2. If a rule does not end with a trailing slash and the user request
the page with a trailing slash, a 404 not found is raised.
This is consistent with how web servers deal with static files. This
also makes it possible to use relative link targets safely.
The :meth:`route` decorator accepts a couple of other arguments
as well:
:param rule: the URL rule as string
:param methods: a list of methods this rule should be limited
to (``GET``, ``POST`` etc.). By default a rule
just listens for ``GET`` (and implicitly ``HEAD``).
:param subdomain: specifies the rule for the subdoain in case
subdomain matching is in use.
:param strict_slashes: can be used to disable the strict slashes
setting for this rule. See above.
:param options: other options to be forwarded to the underlying
:class:`~werkzeug.routing.Rule` object.
"""
def decorator(f):
self.add_url_rule(rule, None, f, **options)
return f
return decorator
def errorhandler(self, code):
"""A decorator that is used to register a function give a given
error code. Example::
@app.errorhandler(404)
def page_not_found():
return 'This page does not exist', 404
You can also register a function as error handler without using
the :meth:`errorhandler` decorator. The following example is
equivalent to the one above::
def page_not_found():
return 'This page does not exist', 404
app.error_handlers[404] = page_not_found
:param code: the code as integer for the handler
"""
def decorator(f):
self.error_handlers[code] = f
return f
return decorator
def template_filter(self, name=None):
"""A decorator that is used to register custom template filter.
You can specify a name for the filter, otherwise the function
name will be used. Example::
@app.template_filter()
def reverse(s):
return s[::-1]
:param name: the optional name of the filter, otherwise the
function name will be used.
"""
def decorator(f):
self.jinja_env.filters[name or f.__name__] = f
return f
return decorator
def before_request(self, f):
"""Registers a function to run before each request."""
self.before_request_funcs.setdefault(None, []).append(f)
return f
def after_request(self, f):
"""Register a function to be run after each request."""
self.after_request_funcs.setdefault(None, []).append(f)
return f
def context_processor(self, f):
"""Registers a template context processor function."""
self.template_context_processors[None].append(f)
return f
def dispatch_request(self):
"""Does the request dispatching. Matches the URL and returns the
return value of the view or error handler. This does not have to
be a response object. In order to convert the return value to a
proper response object, call :func:`make_response`.
"""
req = _request_ctx_stack.top.request
try:
if req.routing_exception is not None:
raise req.routing_exception
return self.view_functions[req.endpoint](**req.view_args)
except HTTPException, e:
handler = self.error_handlers.get(e.code)
if handler is None:
return e
return handler(e)
except Exception, e:
handler = self.error_handlers.get(500)
if self.debug or handler is None:
raise
return handler(e)
from werkzeug.routing import Map, Rule
from sqlalchemy import MetaData
from sqlalchemy.orm import create_session, scoped_session
TEMPLATE_PATH = path.join(path.dirname(__file__), 'templates')
STATIC_PATH = path.join(path.dirname(__file__), 'static')
ALLOWED_SCHEMES = frozenset(['http', 'https', 'ftp', 'ftps'])
URL_CHARS = 'abcdefghijkmpqrstuvwxyzABCDEFGHIJKLMNPQRST23456789'
local = Local()
local_manager = LocalManager([local])
application = local('application')
metadata = MetaData()
url_map = Map([Rule('/static/<file>', endpoint='static', build_only=True)])
session = scoped_session(lambda: create_session(application.database_engine,
autocommit=False,
autoflush=False))
jinja_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH))
def expose(rule, **kw):
def decorate(f):
kw['endpoint'] = f.__name__
url_map.add(Rule(rule, **kw))
return f
return decorate
def __init__(self, import_name):
_PackageBoundObject.__init__(self, import_name)
#: the debug flag. Set this to `True` to enable debugging of
#: the application. In debug mode the debugger will kick in
#: when an unhandled exception ocurrs and the integrated server
#: will automatically reload the application if changes in the
#: code are detected.
self.debug = False
#: a dictionary of all view functions registered. The keys will
#: be function names which are also used to generate URLs and
#: the values are the function objects themselves.
#: to register a view function, use the :meth:`route` decorator.
self.view_functions = {}
#: a dictionary of all registered error handlers. The key is
#: be the error code as integer, the value the function that
#: should handle that error.
#: To register a error handler, use the :meth:`errorhandler`
#: decorator.
self.error_handlers = {}
#: a dictionary with lists of functions that should be called at the
#: beginning of the request. The key of the dictionary is the name of
#: the module this function is active for, `None` for all requests.
#: This can for example be used to open database connections or
#: getting hold of the currently logged in user. To register a
#: function here, use the :meth:`before_request` decorator.
self.before_request_funcs = {}
#: a dictionary with lists of functions that should be called after
#: each request. The key of the dictionary is the name of the module
#: this function is active for, `None` for all requests. This can for
#: example be used to open database connections or getting hold of the
#: currently logged in user. To register a function here, use the
#: :meth:`before_request` decorator.
self.after_request_funcs = {}
#: a dictionary with list of functions that are called without arguments
#: to populate the template context. They key of the dictionary is the
#: name of the module this function is active for, `None` for all
#: requests. Each returns a dictionary that the template context is
#: updated with. To register a function here, use the
#: :meth:`context_processor` decorator.
self.template_context_processors = {
None: [_default_template_ctx_processor]
}
#: the :class:`~werkzeug.routing.Map` for this instance. You can use
#: this to change the routing converters after the class was created
#: but before any routes are connected. Example::
#:
#: from werkzeug import BaseConverter
#:
#: class ListConverter(BaseConverter):
#: def to_python(self, value):
#: return value.split(',')
#: def to_url(self, values):
#: return ','.join(BaseConverter.to_url(value)
#: for value in values)
#:
#: app = Flask(__name__)
#: app.url_map.converters['list'] = ListConverter
self.url_map = Map()
if self.static_path is not None:
self.add_url_rule(self.static_path + '/<filename>',
build_only=True, endpoint='static')
if pkg_resources is not None:
target = (self.import_name, 'static')
else:
target = os.path.join(self.root_path, 'static')
self.wsgi_app = SharedDataMiddleware(self.wsgi_app, {
self.static_path: target
})
#: the Jinja2 environment. It is created from the
#: :attr:`jinja_options` and the loader that is returned
#: by the :meth:`create_jinja_loader` function.
self.jinja_env = Environment(loader=self.create_jinja_loader(),
**self.jinja_options)
self.jinja_env.globals.update(
url_for=url_for,
get_flashed_messages=get_flashed_messages
)
self.jinja_env.filters['tojson'] = _tojson_filter
class ForecastResource(AccountsRequired, RESTResource):
url_map = Map([Rule('/', endpoint='index')], strict_slashes=False)
parents = {
UserResource: '/<string:id_user>/forecast',
MeUserResource: '/forecast',
}
def projection_categories(self, query, categories, start_date, end_date,
income):
s = Decimal('0')
for cluster in query(TransactionsCluster):
if not cluster.enabled or (income is not None and
((cluster.mean_amount < 0) == income)):
continue
transactions = query(Transaction).filter(
Transaction.id_cluster == cluster.id,
Transaction.rdate.between(start_date, end_date))
tables = []
for tr in transactions:
table = {
'text': tr.wording,
'tdate': tr.rdate,
'debited': True,
'disabled': False,
'thisMonth': True,
'category': cluster.category,
'value': tr.value,
'cluster': cluster,
}
tables.append(table)
current_date = cluster.next_date
interval = None
if cluster.median_increment:
interval = timedelta(days=cluster.median_increment)
today = datetime.date.today()
while (interval and current_date <= end_date) or len(tables) == 0:
disabled = False
if current_date and current_date < today:
if (today - current_date).days > (
cluster.median_increment /
2 if cluster.median_increment else 3):
disabled = True
else:
current_date = today + timedelta(days=1)
table = {
'text': cluster.wording,
'tdate': current_date if cluster.next_date else None,
'debited': False,
'thisMonth': current_date and current_date <= end_date,
'disabled': disabled,
'category': cluster.category,
'value': cluster.mean_amount,
'cluster': cluster,
}
tables.append(table)
if not disabled and table['thisMonth']:
s += int(round(cluster.mean_amount))
if interval:
current_date += interval
else:
break
for table in tables:
if cluster.id_category in categories:
categories[cluster.id_category].append(table)
else:
categories[cluster.id_category] = [table]
return s
def compute_projection(self, query, start_date, end_date, income=False):
projection = {
'categories': {},
}
sumA = self.projection_categories(query, projection['categories'],
start_date, end_date, income)
projection['catsum'] = int(round(abs(sumA)))
predSumL = predSumH = predSumA = Decimal('0')
for p in query(Prediction).filter(
Prediction.day == datetime.date.today().day):
if income:
continue
predSumL += p.mean_amount - p.std_amount
predSumH += p.mean_amount + p.std_amount
predSumA += p.mean_amount
sumA += p.mean_amount
projection['predsumL'] = int(round(abs(predSumL)))
projection['predsumH'] = int(round(abs(predSumH)))
projection['predsumA'] = int(round(abs(predSumA)))
projection['sum'] = int(round(abs(sumA)))
return projection
def get_collection(self, request, query, **values):
query.add_filter((Account.display == True)
& (Account.disabled == None))
query.add_join(Account)
today = datetime.date.today()
start_date = first_month_day(today)
end_date = last_month_day(start_date)
tq = query(func.sum(Transaction.value)).filter(
Transaction.application_date.between(start_date, end_date),
Transaction.deleted == None)
actualExpenses = -(tq.filter(Transaction.value < 0).scalar()
or Decimal('0'))
actualIncomes = tq.filter(
Transaction.value >= 0).scalar() or Decimal('0')
balance = query(func.sum(Account.balance),
Account).scalar() or Decimal('0')
anticipated_balance = balance + (query(func.sum(
Transaction.value)).filter(
Transaction.coming == True, Transaction.rdate < end_date,
Transaction.deleted == None).scalar() or Decimal('0'))
income = self.compute_projection(query, start_date, end_date, True)
outcome = self.compute_projection(query, start_date, end_date, False)
balanceL = anticipated_balance + income['catsum'] + income[
'predsumL'] - outcome['catsum'] - outcome['predsumL']
balanceH = anticipated_balance + income['catsum'] + income[
'predsumH'] - outcome['catsum'] - outcome['predsumH']
balanceA = anticipated_balance + income['catsum'] + income[
'predsumA'] - outcome['catsum'] - outcome['predsumA']
restL = (actualIncomes + income['catsum'] + income['predsumL']) - (
actualExpenses + outcome['catsum'] + outcome['predsumL'])
restH = (actualIncomes + income['catsum'] + income['predsumH']) - (
actualExpenses + outcome['catsum'] + outcome['predsumH'])
restA = (actualIncomes + income['catsum'] + income['predsumA']) - (
actualExpenses + outcome['catsum'] + outcome['predsumA'])
response = {
'actualExpenses': -int(round(actualExpenses)),
'anticipatedExpenses': -int(round(outcome['sum'])),
'actualIncomes': int(round(actualIncomes)),
'anticipatedIncomes': int(round(income['sum'])),
'actualBalance': int(round(anticipated_balance)),
'anticipatedBalanceMin': int(round(balanceL)),
'anticipatedBalanceMax': int(round(balanceH)),
'anticipatedBalance': int(round(balanceA)),
'actualResult': int(round(actualIncomes - actualExpenses)),
'anticipatedResultMin': int(round(restL)),
'anticipatedResultMax': int(round(restH)),
'anticipatedResult': int(round(restA)),
'anticipatedVarExpensesMin': -int(round(outcome['predsumL'])),
'anticipatedVarExpensesMax': -int(round(outcome['predsumH'])),
'anticipatedVarExpenses': -int(round(outcome['predsumA'])),
'totalFixedExpenses': -int(round(outcome['catsum'])),
'expenses': self.format_clusters(outcome['categories']),
'anticipatedVarIncomesMin': int(round(income['predsumL'])),
'anticipatedVarIncomesMax': int(round(income['predsumH'])),
'anticipatedVarIncomes': int(round(income['predsumA'])),
'totalFixedIncomes': int(round(income['catsum'])),
'incomes': self.format_clusters(income['categories']),
'anticipatedValuesDate': end_date,
'balances': [],
}
start_date = today - timedelta(days=2 * 30)
end_date = today + timedelta(days=1 * 30)
predictions = {
r[0]: {
'mean_amount': r[1],
'std_amount': r[2]
}
for r in query(Prediction.day, func.sum(Prediction.mean_amount),
func.sum(Prediction.std_amount)).group_by(
Prediction.day)
}
date = start_date
while date <= end_date:
data = {}
data['date'] = date
data['value'] = balance - (query(
func.sum(Transaction.value)).filter(
Transaction.rdate > date, Transaction.coming == False,
Transaction.deleted == None).scalar() or Decimal('0'))
data['value'] += query(func.sum(Transaction.value)).filter(
Transaction.coming == True, Transaction.rdate < date,
Transaction.deleted == None).scalar() or Decimal('0')
if date > today:
bmin = bmax = data['value']
d = today
last_pred = None
while d < date:
try:
pred = predictions[d.day - 1]
except KeyError:
pred = {
'mean_amount': Decimal('0.0'),
'std_amount': Decimal('0.0')
}
if last_pred is not None:
if d.day != 1:
bmin += (
last_pred['mean_amount'] - pred['mean_amount']
) - (last_pred['std_amount'] - pred['std_amount'])
bmax += (
last_pred['mean_amount'] - pred['mean_amount']
) + (last_pred['std_amount'] - pred['std_amount'])
last_pred = pred
for lines in chain(outcome['categories'].itervalues(),
income['categories'].itervalues()):
for line in lines:
if not line['tdate']:
continue
if not line['disabled'] and not line[
'debited'] and line['tdate'] == d:
bmin += line['value']
bmax += line['value']
d += timedelta(days=1)
data['value'] = (bmin + bmax) / 2
data['value'] = int(round(data['value']))
response['balances'].append(data)
date += timedelta(days=2)
first_date, last_date = query(func.min(Transaction.date),
func.max(Transaction.date)).one()
if not all([first_date, last_date
]) or (last_date - first_date).days < 90:
response['warning'] = True
return JsonResponse(response)
def format_clusters(self, categories):
operations = []
for id, ops in categories.iteritems():
if not id:
id = 9998
for op in ops:
if not op['tdate']:
continue
no = {
'wording': op['text'],
'value': int(round(op['cluster'].mean_amount)),
'date': op['tdate'],
'done': op['debited'],
'category': None,
}
if op['category']:
no['category'] = {
'id': op['category'].id,
'name': op['category'].name,
'color': op['category'].color,
}
else:
no['category'] = {
'id': 9998,
'name': u'Indéfini',
'color': 'D7D3BC',
}
operations.append(no)
operations.sort(key=lambda no: no['date'])
return operations
class Builder(object):
default_ignores = ('.*', '_*', 'config.yml', 'Makefile', 'README')
default_programs = {'*.rst': 'rst'}
default_template_path = '_templates'
default_static_folder = 'static'
def __init__(self, project_folder, config):
self.project_folder = os.path.abspath(project_folder)
self.config = config
self.programs = builtin_programs.copy()
self.modules = []
self.storage = {}
self.url_map = Map()
parsed = urlparse(self.config.root_get('canonical_url'))
self.prefix_path = parsed.path
self.url_adapter = self.url_map.bind('dummy.invalid',
script_name=self.prefix_path)
self.register_url('page', '/<path:slug>')
template_path = os.path.join(
self.project_folder,
self.config.root_get('template_path')
or self.default_template_path)
self.locale = Locale(self.config.root_get('locale') or 'en')
self.jinja_env = Environment(
loader=FileSystemLoader([template_path, builtin_templates]),
autoescape=self.config.root_get('template_autoescape', True),
extensions=['jinja2.ext.autoescape', 'jinja2.ext.with_'],
)
self.jinja_env.globals.update(link_to=self.link_to,
format_datetime=self.format_datetime,
format_date=self.format_date,
format_time=self.format_time)
self.static_folder = self.config.root_get('static_folder') or \
self.default_static_folder
for module in self.config.root_get('active_modules') or []:
mod = find_module(module)
mod.setup(self)
self.modules.append(mod)
@property
def default_output_folder(self):
return os.path.join(
self.project_folder,
self.config.root_get('output_folder') or OUTPUT_FOLDER)
def link_to(self, _key, **values):
return self.url_adapter.build(_key, values)
def get_link_filename(self, _key, **values):
link = url_unquote(self.link_to(_key,
**values).lstrip('/')).encode('utf-8')
if not link or link.endswith('/'):
link += 'index.html'
return os.path.join(self.default_output_folder, link)
def open_link_file(self, _key, mode='w', **values):
filename = self.get_link_filename(_key, **values)
folder = os.path.dirname(filename)
if not os.path.isdir(folder):
os.makedirs(folder)
return open(filename, mode)
def register_url(self,
key,
rule=None,
config_key=None,
config_default=None,
**extra):
if config_key is not None:
rule = self.config.root_get(config_key, config_default)
self.url_map.add(Rule(rule, endpoint=key, **extra))
def get_full_static_filename(self, filename):
return os.path.join(self.default_output_folder, self.static_folder,
filename)
def get_static_url(self, filename):
return '/' + posixpath.join(self.static_folder, filename)
def open_static_file(self, filename, mode='w'):
full_filename = self.get_full_static_filename(filename)
folder = os.path.dirname(full_filename)
if not os.path.isdir(folder):
os.makedirs(folder)
return open(full_filename, mode)
def get_storage(self, module):
return self.storage.setdefault(module, {})
def filter_files(self, files, config):
patterns = config.merged_get('ignore_files')
if patterns is None:
patterns = self.default_ignores
result = []
for filename in files:
for pattern in patterns:
if fnmatch(filename, pattern):
break
else:
result.append(filename)
return result
def guess_program(self, config, filename):
mapping = config.list_entries('programs') or self.default_programs
for pattern, program_name in mapping.iteritems():
if fnmatch(filename, pattern):
return program_name
return 'copy'
def render_template(self, template_name, context=None):
if context is None:
context = {}
context['builder'] = self
context.setdefault('config', self.config)
tmpl = self.jinja_env.get_template(template_name)
before_template_rendered.send(tmpl, context=context)
return tmpl.render(context)
def format_datetime(self, datetime=None, format='medium'):
return dates.format_datetime(datetime, format, locale=self.locale)
def format_time(self, time=None, format='medium'):
return dates.format_time(time, format, locale=self.locale)
def format_date(self, date=None, format='medium'):
return dates.format_date(date, format, locale=self.locale)
def iter_contexts(self, prepare=True):
last_config = self.config
cutoff = len(self.project_folder) + 1
for dirpath, dirnames, filenames in os.walk(self.project_folder):
local_config = last_config
local_config_filename = os.path.join(dirpath, 'config.yml')
if os.path.isfile(local_config_filename):
with open(local_config_filename) as f:
local_config = last_config.add_from_file(f)
dirnames[:] = self.filter_files(dirnames, local_config)
filenames = self.filter_files(filenames, local_config)
for filename in filenames:
yield Context(self, local_config,
os.path.join(dirpath[cutoff:], filename),
prepare)
def anything_needs_build(self):
for context in self.iter_contexts(prepare=False):
if context.needs_build:
return True
return False
def run(self):
self.storage.clear()
contexts = list(self.iter_contexts())
for context in contexts:
if context.needs_build:
key = context.is_new and 'A' or 'U'
context.run()
print key, context.source_filename
before_build_finished.send(self)
def debug_serve(self, host='127.0.0.1', port=5000):
from rstblog.server import Server
print 'Serving on http://%s:%d/' % (host, port)
try:
Server(host, port, self).serve_forever()
except KeyboardInterrupt:
pass
class Application(object):
def __init__(self, host, routing_rules=[]):
self._host = host
routing_rules.append(
Rule('/<ruleset_name>', endpoint=self._ruleset_definition_request))
routing_rules.append(
Rule('/<ruleset_name>/<sid>', endpoint=self._state_request))
self._url_map = Map(routing_rules)
def _ruleset_definition_request(self, environ, start_response,
ruleset_name):
def encode_promise(obj):
if isinstance(obj, engine.Promise) or hasattr(obj, '__call__'):
return 'function'
raise TypeError(repr(obj) + " is not JSON serializable")
request = Request(environ)
if request.method == 'GET':
result = self._host.get_ruleset(ruleset_name)
return Response(
json.dumps(result.get_definition(),
default=encode_promise))(environ, start_response)
elif request.method == 'POST':
ruleset_definition = json.loads(request.stream.read())
self._host.set_ruleset(ruleset_name, ruleset_definition)
return Response()(environ, start_response)
def _state_request(self, environ, start_response, ruleset_name, sid):
request = Request(environ)
if request.method == 'GET':
result = self._host.get_state(ruleset_name, sid)
return Response(json.dumps(result))(environ, start_response)
elif request.method == 'POST':
message = json.loads(request.stream.read())
message['sid'] = sid
self._host.post(ruleset_name, message)
elif request.method == 'PATCH':
document = json.loads(request.stream.read())
document['id'] = sid
self._host.patch_state(ruleset_name, document)
return Response()(environ, start_response)
def _not_found(self, environ, start_response):
return Exception('File not found')
def __call__(self, environ, start_response):
request = Request(environ)
adapter = self._url_map.bind_to_environ(environ)
try:
endpoint, values = adapter.match()
return endpoint(environ, start_response, **values)
except HTTPException as e:
return e
def run(self):
self._host.run()
run_simple('127.0.0.1',
5000,
self,
use_debugger=True,
use_reloader=False)
def __init__(self):
self.url_map = Map([])
self.protocols = [JSONProtocol, XMLProtocol]
self.error_handlers = []
from werkzeug.routing import Map, Rule
"""
APP_URLS contain all urls of the app.
"""
APP_URLS = {
'ads_urls':
Map([
Rule('/', endpoint='display_advertisements'),
Rule('/add', endpoint='add_advertisement'),
Rule('/admin', endpoint='get_admin_panel')
]),
}
class Sixpack(object):
def __init__(self, redis_conn):
self.redis = redis_conn
self.config = cfg
self.url_map = Map([
Rule('/', endpoint='home'),
Rule('/_status', endpoint='status'),
Rule('/participate', endpoint='participate'),
Rule('/convert', endpoint='convert'),
Rule('/favicon.ico', endpoint='favicon')
])
def __call__(self, environ, start_response):
return self.wsgi_app(environ, start_response)
def wsgi_app(self, environ, start_response):
request = Request(environ)
response = self.dispatch_request(request)
return response(environ, start_response)
def dispatch_request(self, request):
adapter = self.url_map.bind_to_environ(request.environ)
try:
endpoint, values = adapter.match()
return getattr(self, 'on_' + endpoint)(request, **values)
except NotFound:
return json_error({"message": "not found"}, request, 404)
except HTTPException:
return json_error({"message": "an internal error has occurred"},
request, 500)
@service_unavailable_on_connection_error
def on_status(self, request):
self.redis.ping()
return json_success({'version': __version__}, request)
def on_home(self, request):
dales = """
,-"-.__,-"-.__,-"-..
( C> )( C> )( C> ))
/.`-_-'||`-_-'||`-_-'/
/-"-.--,-"-.--,-"-.--/|
( C> )( C> )( C> )/ |
(|`-_-',.`-_-',.`-_-'/ |
`-----++-----++----'| |
| || || |-'
| || || |
| || || |
`-_-' `-_-' `-_-'
https://github.com/seatgeek/sixpack"""
return Response(dales)
def on_favicon(self, request):
return Response()
@service_unavailable_on_connection_error
def on_convert(self, request):
if should_exclude_visitor(request):
return json_success({'excluded': 'true'}, request)
experiment_name = request.args.get('experiment')
client_id = request.args.get('client_id')
kpi = request.args.get('kpi', None)
if client_id is None or experiment_name is None:
return json_error({'message': 'missing arguments'}, request, 400)
client = Client(client_id, self.redis)
try:
experiment = Experiment.find(experiment_name, self.redis)
if cfg.get('enabled', True):
dt = None
if request.args.get("datetime"):
dt = dateutil.parser.parse(request.args.get("datetime"))
alternative = experiment.convert(client, dt=dt, kpi=kpi)
else:
alternative = experiment.control.name
except ValueError as e:
return json_error({'message': str(e)}, request, 400)
resp = {
'alternative': {
'name': alternative
},
'experiment': {
'name': experiment.name,
},
'conversion': {
'value': None,
'kpi': kpi
},
'client_id': client_id
}
return json_success(resp, request)
@service_unavailable_on_connection_error
def on_participate(self, request):
opts = {}
alts = request.args.getlist('alternatives')
experiment_name = request.args.get('experiment')
force = request.args.get('force')
client_id = request.args.get('client_id')
distribution = request.args.get('traffic_dist')
if client_id is None or experiment_name is None or alts is None:
return json_error({'message': 'missing arguments'}, request, 400)
if distribution:
opts['distribution'] = distribution
try:
experiment = Experiment.find_or_create(experiment_name, alts,
self.redis, opts)
except ValueError as e:
return json_error({'message': str(e)}, request, 400)
alternative = None
if force and force in alts:
alternative = force
elif not cfg.get('enabled', True):
alternative = alts[0]
elif experiment.winner is not None:
alternative = experiment.winner
elif should_exclude_visitor(request):
alternative = alts[0]
else:
dt = None
if request.args.get("datetime"):
dt = dateutil.parser.parse(request.args.get("datetime"))
client = Client(client_id, self.redis)
alternative = experiment.get_alternative(client, dt=dt).name
resp = {
'alternative': {
'name': alternative
},
'experiment': {
'name': experiment.name,
},
'client_id': client_id,
'status': 'ok'
}
return json_success(resp, request)
class Flask(object):
request_class = FlaskRequest
response_class = FlaskResponse
secret_key = 'dangerous'
session_cookie_name = 'flask_session'
def __init__(self):
self.debug = False
self.url_map = Map()
self.view_functions = {}
def dispatch_request(self):
endpoint, values = _request_ctx_stack.top.url_adapter.match()
return self.view_functions[endpoint](**values)
def route(self, rule, **options):
def decorator(f):
print(1)
endpoint = options.setdefault('endpoint', f.__name__)
self.url_map.add(Rule(rule, **options))
self.view_functions[endpoint] = f
return f
return decorator
def open_session(self, request):
return SecureCookie.load_cookie(request,
key=self.session_cookie_name,
secret_key=self.secret_key)
def save_session(self, response):
session.save_cookie(response, key=self.session_cookie_name)
def make_response(self, response):
self.save_session(response)
return response
def request_context(self, environ):
return _RequestContext(self, environ)
def wsgi_app(self, environ, start_response):
with self.request_context(environ):
print('user: %s' % session.setdefault('user', 'none'))
print('total_request: %d' % session.setdefault('total_request', 0))
session['total_request'] += 1
rv = self.dispatch_request()
response = self.response_class(rv)
response = self.make_response(response)
return response(environ, start_response)
def __call__(self, environ, start_response):
print(3)
return self.wsgi_app(environ, start_response)
def run(self, host='localhost', port=5000, **options):
if 'debug' in options:
self.debug = options.pop('debug')
print(2)
options.setdefault('use_reloader', self.debug)
options.setdefault('use_debugger', self.debug)
from werkzeug.serving import run_simple
run_simple(host, port, self, **options)
class CheckmkRESTAPI:
def __init__(self, debug: bool = False):
self.debug = debug
rules = []
for endpoint in ENDPOINT_REGISTRY:
if self.debug:
# This helps us to make sure we can always generate a valid OpenAPI yaml file.
_ = endpoint.to_operation_dict()
rules.append(
Rule(endpoint.default_path,
methods=[endpoint.method],
endpoint=Authenticate(endpoint.wrapped)))
swagger_ui = ServeSwaggerUI(prefix="/[^/]+/check_mk/api/[^/]+/ui")
self.url_map = Map([
Submount(
"/<path:_path>",
[
Rule("/ui/", endpoint=swagger_ui),
Rule("/ui/<path:path>", endpoint=swagger_ui),
Rule("/openapi.yaml", endpoint=swagger_ui.serve_yaml),
Rule("/openapi.json", endpoint=swagger_ui.serve_json),
*rules,
],
),
])
self.wsgi_app = with_context_middleware(
OverrideRequestMethod(self._wsgi_app))
def __call__(self, environ: WSGIEnvironment, start_response):
return self.wsgi_app(environ, start_response)
def _wsgi_app(self, environ: WSGIEnvironment, start_response):
urls = self.url_map.bind_to_environ(environ)
try:
wsgi_app, path_args = urls.match()
# Remove this again (see Submount above), so the validators don't go crazy.
del path_args['_path']
# This is an implicit dependency, as we only know the args at runtime, but the
# function at setup-time.
environ[ARGS_KEY] = path_args
return wsgi_app(environ, start_response)
except HTTPException as exc:
# We don't want to log explicit HTTPExceptions as these are intentional.
# HTTPExceptions are WSGI apps
return exc(environ, start_response)
except MKException as exc:
if self.debug:
raise
return problem(
status=EXCEPTION_STATUS.get(type(exc), 500),
title=str(exc),
detail="An exception occurred.",
)(environ, start_response)
except Exception as exc:
crash = APICrashReport.from_exception()
crash_reporting.CrashReportStore().save(crash)
logger.exception("Unhandled exception (Crash-ID: %s)",
crash.ident_to_text())
if self.debug:
raise
crash_url = f"/{config.omd_site()}/check_mk/crash.py?" + urllib.parse.urlencode(
[
("crash_id", crash.ident_to_text()),
("site", config.omd_site()),
], )
return problem(
status=EXCEPTION_STATUS.get(type(exc), 500),
title=str(exc),
detail=
"An internal error occured while processing your request.",
ext={
'crash_report': {
'href': crash_url,
'method': 'get',
'rel': 'cmk/crash-report',
'type': 'text/html',
},
'crash_id': crash.ident_to_text(),
})(environ, start_response)
def test_allowed_methods_querying():
"""Make sure it's possible to test for allowed methods"""
m = Map([Rule('/<foo>', methods=['GET', 'HEAD']),
Rule('/foo', methods=['POST'])])
a = m.bind('example.org')
assert sorted(a.allowed_methods('/foo')) == ['GET', 'HEAD', 'POST']
def __init__(self):
self.debug = False
self.url_map = Map()
self.view_functions = {}
