def test_resource(self, mock_kr):
"""
L{Klien.resource} returns a L{KleinResource}.
"""
app = Klein()
resource = app.resource()
mock_kr.assert_called_with(app)
self.assertEqual(mock_kr.return_value, resource)
def test_runTCP6(self, reactor, mock_sfs, mock_log, mock_kr):
"""
L{Klein.run} called with tcp6 endpoint description.
"""
app = Klein()
interface = "2001\:0DB8\:f00e\:eb00\:\:1"
spec = "tcp6:8080:interface={0}".format(interface)
app.run(endpoint_description=spec)
reactor.run.assert_called_with()
mock_sfs.assert_called_with(reactor, spec)
mock_log.startLogging.assert_called_with(sys.stdout)
mock_kr.assert_called_with(app)
def test_private_not_visible(self):
"""
When an endpoint is decorated with ``@private_api`` it is
omitted from result of ``makeRst``.
"""
app = Klein()
@private_api
def g():
pass
app.route(b"/g", methods=[b"GET"])(g)
rest = list(makeRst(b"/", "section", app, None, {}))
self.assertEqual(rest, [])
def test_runSSL(self, reactor, mock_sfs, mock_log, mock_kr):
"""
L{Klein.run} called with SSL endpoint specification.
"""
app = Klein()
key = "key.pem"
cert = "cert.pem"
dh_params = "dhparam.pem"
spec_template = "ssl:443:privateKey={0}:certKey={1}"
spec = spec_template.format(key, cert, dh_params)
app.run(endpoint_description=spec)
reactor.run.assert_called_with()
mock_sfs.assert_called_with(reactor, spec)
mock_log.startLogging.assert_called_with(sys.stdout)
mock_kr.assert_called_with(app)
def test_error_handlers_list_is_copied(self):
"""
L{Klein.__copy__} returns a new L{Klein} with all the error handlers
"""
app = Klein()
app.handle_errors(ValueError)(lambda request, failure: 'foo')
app_copy = copy.copy(app)
self.assertEquals(app._error_handlers, app_copy._error_handlers)
app.handle_errors(KeyError)(lambda request, failure: 'foo')
self.assertNotEquals(app._error_handlers, app_copy._error_handlers)
def test_route(self):
"""
L{Klein.route} adds functions as routable endpoints.
"""
app = Klein()
@app.route("/foo")
def foo(request):
return "foo"
c = app.url_map.bind("foo")
self.assertEqual(c.match("/foo"), ("foo", {}))
self.assertEqual(len(app.endpoints), 1)
self.assertEqual(app.execute_endpoint("foo", DummyRequest(1)), "foo")
def test_runWithLogFile(self, reactor, mock_log, mock_site, mock_kr):
"""
L{Klein.run} logs to the specified C{logFile}.
"""
app = Klein()
logFile = Mock()
app.run("localhost", 8080, logFile=logFile)
reactor.listenTCP.assert_called_with(8080, mock_site.return_value, interface="localhost")
reactor.run.assert_called_with()
mock_site.assert_called_with(mock_kr.return_value)
mock_kr.assert_called_with(app)
mock_log.startLogging.assert_called_with(logFile)
def test_run(self, reactor, mock_log, mock_site, mock_kr):
"""
L{Klein.run} configures a L{KleinResource} and a L{Site}
listening on the specified interface and port, and logs
to stdout.
"""
app = Klein()
app.run("localhost", 8080)
reactor.listenTCP.assert_called_with(8080, mock_site.return_value, interface="localhost")
reactor.run.assert_called_with()
mock_site.assert_called_with(mock_kr.return_value)
mock_kr.assert_called_with(app)
mock_log.startLogging.assert_called_with(sys.stdout)
def webserver_for_test(test, url_path, response_content):
"""
Create a webserver that serves ``response_content`` from ``url_path``.
"""
app = Klein()
@app.route(url_path)
def _respond(request):
return response_content
factory = Site(app.resource())
endpoint = serverFromString(reactor, b"tcp:0")
listening = endpoint.listen(factory)
def stop_port(port):
test.addCleanup(port.stopListening)
return port
listening.addCallback(stop_port)
return listening
def test_submountedRoute(self):
"""
L{Klein.subroute} adds functions as routable endpoints.
"""
app = Klein()
with app.subroute("/sub") as app:
@app.route("/prefixed_uri")
def foo_endpoint(request):
return b"foo"
c = app.url_map.bind("sub/prefixed_uri")
self.assertEqual(
c.match("/sub/prefixed_uri"), ("foo_endpoint", {}))
self.assertEqual(
len(app.endpoints), 1)
self.assertEqual(
app.execute_endpoint("foo_endpoint", DummyRequest(1)), b"foo")
def test_stackedRoute(self):
"""
L{Klein.route} can be stacked to create multiple endpoints of
a single function.
"""
app = Klein()
@app.route("/foo")
@app.route("/bar", endpoint="bar")
def foobar(request):
return "foobar"
self.assertEqual(len(app.endpoints), 2)
c = app.url_map.bind("foo")
self.assertEqual(c.match("/foo"), ("foobar", {}))
self.assertEqual(app.execute_endpoint("foobar", DummyRequest(1)), "foobar")
self.assertEqual(c.match("/bar"), ("bar", {}))
self.assertEqual(app.execute_endpoint("bar", DummyRequest(2)), "foobar")
def test_routes(self):
"""
L{getRoutes} returns all the defined routes and their attributes.
"""
app = Klein()
def f():
pass
f.attr = "attr"
def g():
pass
g.attr = "G"
app.route(b"/", methods=[b"GET"])(f)
app.route(b"/g", methods=[b"PUT", b"OPTIONS"])(g)
routes = sorted(getRoutes(app))
self.assertEqual(routes, [
KleinRoute(methods={b"GET"}, path=b"/", endpoint="f",
attributes={'attr': 'attr'}),
KleinRoute(methods={b'PUT', b'OPTIONS'}, path=b'/g', endpoint='g',
attributes={'attr': 'G'})])
def test_copy(self):
"""
L{Klein.__copy__} returns a new L{Klein} with all the registered endpoints
"""
app = Klein()
@app.route("/foo")
def foo(request):
return "foo"
app_copy = copy.copy(app)
@app.route('/bar')
def bar(request):
return 'bar'
dr1 = DummyRequest(1)
dr2 = DummyRequest(2)
dr3 = DummyRequest(3)
self.assertEquals(app.execute_endpoint('foo', dr1), 'foo')
self.assertEquals(app.execute_endpoint('bar', dr2), 'bar')
self.assertRaises(KeyError, app_copy.execute_endpoint, 'bar', dr3)
def main(reactor):
component = Component(
transports=u"ws://localhost:8080/ws",
realm=u"crossbardemo",
)
app = Klein()
webapp = WebApplication(app, component)
# have our Web site listen on 8090
site = Site(app.resource())
server_ep = TCP4ServerEndpoint(reactor, 8090)
port = yield server_ep.listen(site)
print("Web application on {}".format(port))
# we don't *have* to hand over control of the reactor to
# component.run -- if we don't want to, we call .start()
# The Deferred it returns fires when the component is "completed"
# (or errbacks on any problems).
comp_d = component.start(reactor)
# When not using run() we also must start logging ourselves.
import txaio
txaio.start_logging(level='info')
# If the Component raises an exception we want to exit. Note that
# things like failing to connect will be swallowed by the
# re-connection mechanisms already so won't reach here.
def _failed(f):
print("Component failed: {}".format(f))
done.errback(f)
comp_d.addErrback(_failed)
# wait forever (unless the Component raises an error)
done = Deferred()
yield done
import cgi
from urlparse import parse_qsl
from klein import Klein
from twisted.python.components import registerAdapter
from twisted.internet import reactor
from twisted.web.server import Session
from twisted.web.static import File
from zope.interface import Interface, Attribute, implements
from jinja2 import Environment, FileSystemLoader
from mailclient import sendmail
from imap import get_messages
app = Klein()
env = Environment(loader=FileSystemLoader('templates'))
def render_template(filename, **kwargs):
template = env.get_template(filename)
return template.render(**kwargs)
class ILogin(Interface):
email = Attribute("user's email address")
password = Attribute("user's email password")
class Login(object):
implements(ILogin)
def __init__(self, session):
self.email = ''
self.password = ''
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
###############################################################################
from twisted.internet.defer import inlineCallbacks, returnValue
from klein import Klein
from autobahn.twisted.wamp import Application
app = Klein()
wampapp = Application()
@app.route('/square/submit', methods=['POST'])
@inlineCallbacks
def square_submit(request):
x = int(request.args.get('x', [0])[0])
res = yield wampapp.session.call(u'com.example.square', x)
returnValue("{} squared is {}".format(x, res))
if __name__ == "__main__":
import sys
from twisted.python import log
from twisted.web.server import Site
class _Another(object):
anotherKlein = Klein()
@anotherKlein.route("/bar")
def bar(self):
pass
class RasaCoreServer(object):
"""Class representing a Rasa Core HTTP server."""
app = Klein()
def __init__(self,
model_directory,
interpreter=None,
loglevel="INFO",
logfile="rasa_core.log",
cors_origins=None,
action_factory=None,
auth_token=None,
tracker_store=None):
_configure_logging(loglevel, logfile)
self.config = {
"cors_origins": cors_origins if cors_origins else [],
"token": auth_token
}
self.model_directory = model_directory
self.interpreter = interpreter
self.tracker_store = tracker_store
self.action_factory = action_factory
self.agent = self._create_agent(model_directory, interpreter,
action_factory, tracker_store)
@staticmethod
def _create_agent(
model_directory, # type: Text
interpreter, # type: Union[Text, NaturalLanguageInterpreter]
action_factory=None, # type: Optional[Text]
tracker_store=None # type: Optional[TrackerStore]
):
# type: (...) -> Optional[Agent]
try:
return Agent.load(model_directory,
interpreter,
tracker_store=tracker_store,
action_factory=action_factory)
except Exception as e:
logger.warn("Failed to load any agent model. Running "
"Rasa Core server with out loaded model now. {}"
"".format(e))
return None
@app.route("/", methods=['GET', 'OPTIONS'])
@check_cors
def hello(self, request):
"""Check if the server is running and responds with the version."""
return "hello from Rasa Core: " + __version__
@app.route("/conversations/<sender_id>/continue",
methods=['POST', 'OPTIONS'])
@check_cors
@requires_auth
@ensure_loaded_agent
def continue_predicting(self, request, sender_id):
"""Continue a prediction started with parse.
Caller should have executed the action returned from the parse
endpoint. The events returned from that executed action are
passed to continue which will trigger the next action prediction.
If continue predicts action listen, the caller should wait for the
next user message."""
request.setHeader('Content-Type', 'application/json')
request_params = json.loads(request.content.read().decode(
'utf-8', 'strict'))
encoded_events = request_params.get("events", [])
executed_action = request_params.get("executed_action", None)
evts = events.deserialise_events(encoded_events)
try:
response = self.agent.continue_message_handling(
sender_id, executed_action, evts)
except ValueError as e:
request.setResponseCode(400)
return json.dumps({"error": e.message})
except Exception as e:
request.setResponseCode(500)
logger.exception(e)
return json.dumps({"error": "Server failure. Error: {}".format(e)})
return json.dumps(response)
@app.route("/conversations/<sender_id>/tracker/events",
methods=['POST', 'OPTIONS'])
@check_cors
@ensure_loaded_agent
def append_events(self, request, sender_id):
"""Append a list of events to the state of a conversation"""
request.setHeader('Content-Type', 'application/json')
request_params = json.loads(request.content.read().decode(
'utf-8', 'strict'))
evts = events.deserialise_events(request_params)
tracker = self.agent.tracker_store.get_or_create_tracker(sender_id)
for e in evts:
tracker.update(e)
self.agent.tracker_store.save(tracker)
return json.dumps(tracker.current_state())
@app.route("/conversations/<sender_id>/tracker",
methods=['GET', 'OPTIONS'])
@check_cors
@ensure_loaded_agent
def retrieve_tracker(self, request, sender_id):
"""Get a dump of a conversations tracker including its events."""
# parameters
use_history = bool_arg(request, 'ignore_restarts', default=False)
should_include_events = bool_arg(request, 'events', default=True)
until_time = request.args.get('until', None)
# retrieve tracker and set to requested state
tracker = self.agent.tracker_store.get_or_create_tracker(sender_id)
if until_time is not None:
tracker = tracker.travel_back_in_time(float(until_time))
# dump and return tracker
state = tracker.current_state(
should_include_events=should_include_events,
only_events_after_latest_restart=use_history)
return json.dumps(state)
@app.route("/conversations/<sender_id>/tracker",
methods=['PUT', 'OPTIONS'])
@check_cors
@ensure_loaded_agent
def update_tracker(self, request, sender_id):
"""Use a list of events to set a conversations tracker to a state."""
request.setHeader('Content-Type', 'application/json')
request_params = json.loads(request.content.read().decode(
'utf-8', 'strict'))
tracker = DialogueStateTracker.from_dict(sender_id, request_params,
self.agent.domain)
self.agent.tracker_store.save(tracker)
# will override an existing tracker with the same id!
self.agent.tracker_store.save(tracker)
return json.dumps(tracker.current_state(should_include_events=True))
@app.route("/conversations/<sender_id>/parse",
methods=['GET', 'POST', 'OPTIONS'])
@check_cors
@requires_auth
@ensure_loaded_agent
def parse(self, request, sender_id):
request.setHeader('Content-Type', 'application/json')
if request.method.decode('utf-8', 'strict') == 'GET':
request_params = {
key.decode('utf-8', 'strict'):
value[0].decode('utf-8', 'strict')
for key, value in request.args.items()
}
else:
content = request.content.read()
try:
request_params = json.loads(content.decode('utf-8', 'strict'))
except ValueError as e:
logger.error("Failed to decode json during parse request. "
"Error: {}. Request content: "
"'{}'".format(e, content))
raise
if 'query' in request_params:
message = request_params.pop('query')
elif 'q' in request_params:
message = request_params.pop('q')
else:
request.setResponseCode(400)
return json.dumps({"error": "Invalid parse parameter specified"})
try:
response = self.agent.start_message_handling(message, sender_id)
request.setResponseCode(200)
return json.dumps(response)
except Exception as e:
request.setResponseCode(500)
logger.error("Caught an exception during "
"parse: {}".format(e),
exc_info=1)
return json.dumps({"error": "{}".format(e)})
@app.route("/load", methods=['POST', 'OPTIONS'])
@check_cors
def load_model(self, request):
"""Loads a zipped model, replacing the existing one."""
logger.info("Received new model through REST interface.")
zipped_path = tempfile.NamedTemporaryFile(delete=False, suffix=".zip")
zipped_path.close()
with io.open(zipped_path.name, 'wb') as f:
f.write(request.args[b'model'][0])
logger.debug("Downloaded model to {}".format(zipped_path.name))
zip_ref = zipfile.ZipFile(zipped_path.name, 'r')
zip_ref.extractall(self.model_directory)
zip_ref.close()
logger.debug("Unzipped model to {}".format(
os.path.abspath(self.model_directory)))
self.agent = self._create_agent(self.model_directory, self.interpreter,
self.action_factory,
self.tracker_store)
logger.debug("Finished loading new agent.")
return json.dumps({'success': 1})
@app.route("/version", methods=['GET', 'OPTIONS'])
@check_cors
def version(self, request):
"""respond with the version number of the installed rasa core."""
request.setHeader('content-type', 'application/json')
return json.dumps({'version': __version__})
import json
from itertools import islice
from ledStrip import ledstrip
from threading import Thread
import Queue
from collections import deque
from time import sleep
from klein import Klein
from twisted.internet import reactor
BRIGHTNESS=50
SHIFTTIME = 5*60
#app = Flask(__name__)
app = Klein()
@app.route("/")
def hello():
return "Hello World!"
@app.route("/webhook", methods=["POST"])
def webhook(request):
global states
try:
payload = json.loads(request.content.read())
state = payload.get('details', {}).get('state')
print 'got state: ', state
states.append(state.lower())
states.popleft()
turnOnTheLights(states, timeoutFlag = True)
import json
import datetime
from twisted.internet import reactor
from twisted.internet.defer import inlineCallbacks, returnValue
from klein import Klein
from bson import ObjectId
from txmongo import MongoConnection
from klein_babel import gettext, locale_from_request
from umongo import Instance, Document, fields, ValidationError, set_gettext
from umongo.marshmallow_bonus import SchemaFromUmongo
app = Klein()
db = MongoConnection().demo_umongo
instance = Instance(db)
set_gettext(gettext)
class MongoJsonEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, (datetime.datetime, datetime.date)):
return obj.isoformat()
elif isinstance(obj, ObjectId):
return str(obj)
return json.JSONEncoder.default(self, obj)
def jsonify(request, *args, **kwargs):
"""
jsonify with support for MongoDB ObjectId
class ConfigurationAPIUserV1(object):
"""
A user accessing the API.
The APIs exposed here typically operate on cluster configuration. They
frequently return success results when a configuration change has been made
durable but has not yet been deployed onto the cluster.
"""
app = Klein()
def __init__(self, persistence_service, cluster_state_service):
"""
:param ConfigurationPersistenceService persistence_service: Service
for retrieving and setting desired configuration.
:param ClusterStateService cluster_state_service: Service that
knows about the current state of the cluster.
"""
self.persistence_service = persistence_service
self.cluster_state_service = cluster_state_service
@app.route("/version", methods=['GET'])
@user_documentation(
u"""
Get the version of Flocker being run.
""",
section=u"common",
header=u"Get Flocker version",
examples=[u"get version"])
@structured(
inputSchema={},
outputSchema={'$ref': '/v1/endpoints.json#/definitions/versions'},
schema_store=SCHEMAS
)
def version(self):
"""
Return the ``flocker`` version string.
"""
return {u"flocker": __version__}
@app.route("/configuration/datasets", methods=['GET'])
@user_documentation(
u"""
Get the cluster's dataset configuration.
""",
header=u"Get the cluster's dataset configuration",
examples=[u"get configured datasets"],
section=u"dataset",
)
@structured(
inputSchema={},
outputSchema={
'$ref':
'/v1/endpoints.json#/definitions/configuration_datasets_list',
},
schema_store=SCHEMAS,
)
def get_dataset_configuration(self):
"""
Get the configured datasets.
:return: A ``list`` of ``dict`` representing each of dataset
that is configured to exist anywhere on the cluster.
"""
return list(datasets_from_deployment(self.persistence_service.get()))
@app.route("/configuration/datasets", methods=['POST'])
@user_documentation(
u"""
Create a new dataset.
""",
header=u"Create new dataset",
examples=[
u"create dataset",
u"create dataset with dataset_id",
u"create dataset with duplicate dataset_id",
u"create dataset with maximum_size",
u"create dataset with metadata",
],
section=u"dataset",
)
@structured(
inputSchema={
'$ref':
'/v1/endpoints.json#/definitions/configuration_datasets_create'
},
outputSchema={
'$ref':
'/v1/endpoints.json#/definitions/configuration_datasets'},
schema_store=SCHEMAS
)
def create_dataset_configuration(self, primary, dataset_id=None,
maximum_size=None, metadata=None):
"""
Create a new dataset in the cluster configuration.
:param unicode primary: The UUID of the node on which the primary
manifestation of the dataset will be created.
:param unicode dataset_id: A unique identifier to assign to the
dataset. This is a string giving a UUID (per RFC 4122). If no
value is given, one will be generated and returned in the response.
This is not for easy human use. For human-friendly identifiers,
use items in ``metadata``.
:param maximum_size: Either the maximum number of bytes the dataset
will be capable of storing or ``None`` to make the dataset size
unlimited. This may be optional or required depending on the
dataset backend.
:param dict metadata: A small collection of unicode key/value pairs to
associate with the dataset. These items are not interpreted. They
are only stored and made available for later retrieval. Use this
for things like human-friendly dataset naming, ownership
information, etc.
:return: A ``dict`` describing the dataset which has been added to the
cluster configuration or giving error information if this is not
possible.
"""
if dataset_id is None:
dataset_id = unicode(uuid4())
dataset_id = dataset_id.lower()
if metadata is None:
metadata = {}
primary = UUID(hex=primary)
# Use persistence_service to get a Deployment for the cluster
# configuration.
deployment = self.persistence_service.get()
for node in deployment.nodes:
for manifestation in node.manifestations.values():
if manifestation.dataset.dataset_id == dataset_id:
raise DATASET_ID_COLLISION
# XXX Check cluster state to determine if the given primary node
# actually exists. If not, raise PRIMARY_NODE_NOT_FOUND.
# See FLOC-1278
dataset = Dataset(
dataset_id=dataset_id,
maximum_size=maximum_size,
metadata=pmap(metadata)
)
manifestation = Manifestation(dataset=dataset, primary=True)
primary_node = deployment.get_node(primary)
new_node_config = primary_node.transform(
("manifestations", manifestation.dataset_id), manifestation)
new_deployment = deployment.update_node(new_node_config)
saving = self.persistence_service.save(new_deployment)
def saved(ignored):
result = api_dataset_from_dataset_and_node(dataset, primary)
return EndpointResponse(CREATED, result)
saving.addCallback(saved)
return saving
@app.route("/configuration/datasets/<dataset_id>", methods=['DELETE'])
@user_documentation(
u"""
Deletion is idempotent: deleting a dataset multiple times will
result in the same response.
""",
header=u"Delete an existing dataset",
examples=[
u"delete dataset",
u"delete dataset with unknown dataset id",
],
section=u"dataset",
)
@structured(
inputSchema={},
outputSchema={
'$ref':
'/v1/endpoints.json#/definitions/configuration_datasets'},
schema_store=SCHEMAS
)
def delete_dataset(self, dataset_id):
"""
Delete an existing dataset in the cluster configuration.
:param unicode dataset_id: The unique identifier of the dataset. This
is a string giving a UUID (per RFC 4122).
:return: A ``dict`` describing the dataset which has been marked
as deleted in the cluster configuration or giving error
information if this is not possible.
"""
# Get the current configuration.
deployment = self.persistence_service.get()
# XXX this doesn't handle replicas
# https://clusterhq.atlassian.net/browse/FLOC-1240
old_manifestation, origin_node = _find_manifestation_and_node(
deployment, dataset_id)
new_node = origin_node.transform(
("manifestations", dataset_id, "dataset", "deleted"), True)
deployment = deployment.update_node(new_node)
saving = self.persistence_service.save(deployment)
def saved(ignored):
result = api_dataset_from_dataset_and_node(
new_node.manifestations[dataset_id].dataset, new_node.uuid,
)
return EndpointResponse(OK, result)
saving.addCallback(saved)
return saving
@app.route("/configuration/datasets/<dataset_id>", methods=['POST'])
@user_documentation(
u"""
This can be used to:
* Move a dataset from one node to another by changing the
``primary`` attribute.
* In the future update metadata.
""",
header=u"Update an existing dataset",
examples=[
u"update dataset with primary",
u"update dataset with unknown dataset id",
],
section=u"dataset",
)
@structured(
inputSchema={
'$ref':
'/v1/endpoints.json#/definitions/configuration_datasets_update'},
outputSchema={
'$ref':
'/v1/endpoints.json#/definitions/configuration_datasets'},
schema_store=SCHEMAS
)
def update_dataset(self, dataset_id, primary=None):
"""
Update an existing dataset in the cluster configuration.
:param unicode dataset_id: The unique identifier of the dataset. This
is a string giving a UUID (per RFC 4122).
:param primary: The UUID of the node to which the dataset will be
moved, or ``None`` indicating no change.
:return: A ``dict`` describing the dataset which has been added to the
cluster configuration or giving error information if this is not
possible.
"""
# Get the current configuration.
deployment = self.persistence_service.get()
# Raises DATASET_NOT_FOUND if the ``dataset_id`` is not found.
primary_manifestation, current_node = _find_manifestation_and_node(
deployment, dataset_id
)
if primary_manifestation.dataset.deleted:
raise DATASET_DELETED
if primary is not None:
deployment = _update_dataset_primary(
deployment, dataset_id, UUID(hex=primary)
)
saving = self.persistence_service.save(deployment)
primary_manifestation, current_node = _find_manifestation_and_node(
deployment, dataset_id
)
# Return an API response dictionary containing the dataset with updated
# primary address.
def saved(ignored):
result = api_dataset_from_dataset_and_node(
primary_manifestation.dataset,
current_node.uuid,
)
return EndpointResponse(OK, result)
saving.addCallback(saved)
return saving
@app.route("/state/datasets", methods=['GET'])
@user_documentation(
u"""
The result reflects the control service's knowledge, which may be
out of date or incomplete. E.g. a dataset agent has not connected
or updated the control service yet.
""",
header=u"Get current cluster datasets",
examples=[u"get state datasets"],
section=u"dataset",
)
@structured(
inputSchema={},
outputSchema={
'$ref': '/v1/endpoints.json#/definitions/state_datasets_array'
},
schema_store=SCHEMAS
)
def state_datasets(self):
"""
Return all primary manifest datasets and all non-manifest datasets in
the cluster.
:return: A ``list`` containing all datasets in the cluster.
"""
# XXX This duplicates code in datasets_from_deployment, but that
# function is designed to operate on a Deployment rather than a
# DeploymentState instance and the dataset configuration result
# includes metadata and deleted flags which should not be part of the
# dataset state response.
# Refactor. See FLOC-2207.
response = []
deployment_state = self.cluster_state_service.as_deployment()
get_manifestation_path = self.cluster_state_service.manifestation_path
for dataset, node in deployment_state.all_datasets():
response_dataset = dict(
dataset_id=dataset.dataset_id,
)
if node is not None:
response_dataset[u"primary"] = unicode(node.uuid)
response_dataset[u"path"] = get_manifestation_path(
node.uuid,
dataset[u"dataset_id"]
).path.decode("utf-8")
if dataset.maximum_size is not None:
response_dataset[u"maximum_size"] = dataset.maximum_size
response.append(response_dataset)
return response
@app.route("/configuration/containers", methods=['GET'])
@user_documentation(
u"""
These containers may or may not actually exist on the
cluster.
""",
header=u"Get the cluster's container configuration",
examples=[u"get configured containers"],
section=u"container",
)
@structured(
inputSchema={},
outputSchema={
'$ref':
'/v1/endpoints.json#/definitions/configuration_containers_array',
},
schema_store=SCHEMAS,
)
def get_containers_configuration(self):
"""
Get the configured containers.
:return: A ``list`` of ``dict`` representing each of the containers
that are configured to exist anywhere on the cluster.
"""
return list(containers_from_deployment(self.persistence_service.get()))
@app.route("/state/containers", methods=['GET'])
@user_documentation(
u"""
This reflects the control service's knowledge of the cluster,
which may be out of date or incomplete, e.g. if a container agent
has not connected or updated the control service yet.
""",
header=u"Get the cluster's actual containers",
examples=[u"get actual containers"],
section=u"container",
)
@structured(
inputSchema={},
outputSchema={
'$ref':
'/v1/endpoints.json#/definitions/state_containers_array',
},
schema_store=SCHEMAS,
)
def get_containers_state(self):
"""
Get the containers present in the cluster.
:return: A ``list`` of ``dict`` representing each of the containers
that are configured to exist anywhere on the cluster.
"""
result = []
deployment_state = self.cluster_state_service.as_deployment()
for node in deployment_state.nodes:
if node.applications is None:
continue
for application in node.applications:
container = container_configuration_response(
application, node.uuid)
container[u"running"] = application.running
result.append(container)
return result
def _get_attached_volume(self, node_uuid, volume):
"""
Create an ``AttachedVolume`` given a volume dictionary.
:param UUID node_uuid: The node where the volume should be.
:param dict volume: Parameters for specific volume passed to creation
endpoint.
:return AttachedVolume: Corresponding instance.
"""
deployment = self.persistence_service.get()
instances = list(manifestations_from_deployment(
deployment, volume[u"dataset_id"]))
if not any(m for (m, _) in instances if not m.dataset.deleted):
raise DATASET_NOT_FOUND
if not any(n for (_, n) in instances if n.uuid == node_uuid):
raise DATASET_ON_DIFFERENT_NODE
if any(app for app in deployment.applications() if
app.volume and
app.volume.manifestation.dataset_id == volume[u"dataset_id"]):
raise DATASET_IN_USE
return AttachedVolume(
manifestation=[m for (m, node) in instances
if node.uuid == node_uuid and m.primary][0],
mountpoint=FilePath(volume[u"mountpoint"].encode("utf-8")))
@app.route("/configuration/containers", methods=['POST'])
@user_documentation(
u"""
The container will be automatically started once it is created on
the cluster.
""",
header=u"Add a new container to the configuration",
examples=[
u"create container",
u"create container with duplicate name",
u"create container with ports",
u"create container with environment",
u"create container with attached volume",
u"create container with cpu shares",
u"create container with memory limit",
u"create container with links",
u"create container with command line",
# No example of creating a container with a different restart
# policy because only the "never" policy is supported. See
# FLOC-2449.
],
section=u"container",
)
@structured(
inputSchema={
'$ref': '/v1/endpoints.json#/definitions/configuration_container'},
outputSchema={
'$ref': '/v1/endpoints.json#/definitions/configuration_container'},
schema_store=SCHEMAS
)
def create_container_configuration(
self, node_uuid, name, image, ports=(), environment=None,
restart_policy=None, cpu_shares=None, memory_limit=None,
links=(), volumes=(), command_line=None,
):
"""
Create a new dataset in the cluster configuration.
:param unicode node_uuid: The UUID of the node on which the container
will run.
:param unicode name: A unique identifier for the container within
the Flocker cluster.
:param unicode image: The name of the Docker image to use for the
container.
:param list ports: A ``list`` of ``dict`` objects, mapping internal
to external ports for the container.
:param dict environment: A ``dict`` of key/value pairs to be supplied
to the container as environment variables. Keys and values must be
``unicode``.
:param dict restart_policy: A restart policy for the container, this
is a ``dict`` with at a minimum a "name" key, whose value must be
one of "always", "never" or "on-failure". If the "name" is given
as "on-failure", there may also be another optional key
"maximum_retry_count", containing a positive ``int`` specifying
the maximum number of times we should attempt to restart a failed
container.
:param volumes: A iterable of ``dict`` with ``"dataset_id"`` and
``"mountpoint"`` keys.
:param int cpu_shares: A positive integer specifying the relative
weighting of CPU cycles for this container (see Docker's run
reference for further information).
:param int memory_limit: A positive integer specifying the maximum
amount of memory in bytes available to this container.
:param list links: A ``list`` of ``dict`` objects, mapping container
links via "alias", "local_port" and "remote_port" values.
:param command_line: If not ``None``, the command line to use when
running the Docker image's entry point.
:return: An ``EndpointResponse`` describing the container which has
been added to the cluster configuration.
"""
deployment = self.persistence_service.get()
node_uuid = UUID(hex=node_uuid)
# Check if container by this name already exists, if it does
# return error.
for node in deployment.nodes:
for application in node.applications:
if application.name == name:
raise CONTAINER_NAME_COLLISION
# Find the volume, if any; currently we only support one volume
# https://clusterhq.atlassian.net/browse/FLOC-49
attached_volume = None
if volumes:
attached_volume = self._get_attached_volume(node_uuid, volumes[0])
# Find the node.
node = deployment.get_node(node_uuid)
# Check if we have any ports in the request. If we do, check existing
# external ports exposed to ensure there is no conflict. If there is a
# conflict, return an error.
for port in ports:
for current_node in deployment.nodes:
for application in current_node.applications:
for application_port in application.ports:
if application_port.external_port == port['external']:
raise CONTAINER_PORT_COLLISION
# If links are present, check that there are no conflicts in local
# ports or alias names.
link_aliases = set()
link_local_ports = set()
application_links = set()
for link in links:
if link['alias'] in link_aliases:
raise LINK_ALIAS_COLLISION
if link['local_port'] in link_local_ports:
raise LINK_PORT_COLLISION
link_aliases.add(link['alias'])
link_local_ports.add(link['local_port'])
application_links.add(
Link(
alias=link['alias'], local_port=link['local_port'],
remote_port=link['remote_port']
)
)
# If we have ports specified, add these to the Application instance.
application_ports = []
for port in ports:
application_ports.append(Port(
internal_port=port['internal'],
external_port=port['external']
))
if environment is not None:
environment = frozenset(environment.items())
if restart_policy is None:
restart_policy = dict(name=u"never")
policy_name = restart_policy.pop("name")
policy_factory = FLOCKER_RESTART_POLICY_NAME_TO_POLICY[policy_name]
policy = policy_factory(**restart_policy)
# Create Application object, add to Deployment, save.
application = Application(
name=name,
image=DockerImage.from_string(image),
ports=frozenset(application_ports),
environment=environment,
volume=attached_volume,
restart_policy=policy,
cpu_shares=cpu_shares,
memory_limit=memory_limit,
links=application_links,
command_line=command_line,
)
new_node_config = node.transform(
["applications"],
lambda s: s.add(application)
)
new_deployment = deployment.update_node(new_node_config)
saving = self.persistence_service.save(new_deployment)
# Return passed in dictionary with CREATED response code.
def saved(_):
result = container_configuration_response(application, node_uuid)
return EndpointResponse(CREATED, result)
saving.addCallback(saved)
return saving
@app.route("/configuration/containers/<name>", methods=['POST'])
@user_documentation(
u"""
This will lead to the container being relocated to the specified host
and restarted. This will also update the primary host of any attached
datasets.
""",
header=u"Update a named container's configuration",
examples=[u"move container"],
section=u"container",
)
@structured(
inputSchema={
'$ref':
'/v1/endpoints.json#/definitions/configuration_container_update',
},
outputSchema={
'$ref':
'/v1/endpoints.json#/definitions/configuration_container',
},
schema_store=SCHEMAS,
)
def update_containers_configuration(self, name, node_uuid):
"""
Update the specified container's configuration.
:param unicode name: A unique identifier for the container within
the Flocker cluster.
:param unicode node_uuid: The address of the node on which the
container will run.
:return: An ``EndpointResponse`` describing the container which has
been updated.
"""
deployment = self.persistence_service.get()
node_uuid = UUID(hex=node_uuid)
target_node = deployment.get_node(node_uuid)
for node in deployment.nodes:
for application in node.applications:
if application.name == name:
deployment = deployment.move_application(
application, target_node
)
saving = self.persistence_service.save(deployment)
def saved(_):
result = container_configuration_response(
application, node_uuid
)
return EndpointResponse(OK, result)
saving.addCallback(saved)
return saving
# Didn't find the application:
raise CONTAINER_NOT_FOUND
@app.route("/configuration/containers/<name>", methods=['DELETE'])
@user_documentation(
u"""
This will lead to the container being stopped and not being
restarted again. Any datasets that were attached as volumes will
continue to exist on the cluster.
""",
header=u"Remove a container from the configuration",
examples=[
u"remove a container",
u"remove a container with unknown name",
],
section=u"container",
)
@structured(
inputSchema={},
outputSchema={},
schema_store=SCHEMAS
)
def delete_container_configuration(self, name):
"""
Remove a container from the cluster configuration.
:param unicode name: A unique identifier for the container within
the Flocker cluster.
:return: An ``EndpointResponse``.
"""
deployment = self.persistence_service.get()
for node in deployment.nodes:
for application in node.applications:
if application.name == name:
updated_node = node.transform(
["applications"], lambda s: s.remove(application))
d = self.persistence_service.save(
deployment.update_node(updated_node))
d.addCallback(lambda _: None)
return d
# Didn't find the application:
raise CONTAINER_NOT_FOUND
@app.route("/state/nodes", methods=['GET'])
@user_documentation(
u"""
Some nodes may not be listed if their agents are disconnected from
the cluster. IP addresses may be private IP addresses that are not
publicly routable.
""",
header=u"List known nodes in the cluster",
examples=[
u"list known nodes",
],
section=u"common",
)
@structured(
inputSchema={},
outputSchema={"$ref":
'/v1/endpoints.json#/definitions/nodes_array'},
schema_store=SCHEMAS
)
def list_current_nodes(self):
return [{u"host": node.hostname, u"uuid": unicode(node.uuid)}
for node in
self.cluster_state_service.as_deployment().nodes]
@app.route("/configuration/_compose", methods=['POST'])
@private_api
@structured(
inputSchema={
'$ref':
'/v1/endpoints.json#/definitions/configuration_compose'
},
outputSchema={},
schema_store=SCHEMAS
)
def replace_configuration(self, applications, deployment):
"""
Replace the existing configuration with one given by flocker-deploy
command line tool.
:param applications: Configuration in Flocker-native or
Fig/Compose format.
:param deployment: Configuration of which applications run on
which nodes.
"""
try:
configuration = FigConfiguration(applications)
if not configuration.is_valid_format():
configuration = FlockerConfiguration(applications)
return self.persistence_service.save(model_from_configuration(
deployment_state=self.cluster_state_service.as_deployment(),
applications=configuration.applications(),
deployment_configuration=deployment))
except ConfigurationError as e:
raise make_bad_request(code=BAD_REQUEST, description=unicode(e))
class RasaNLU(object):
"""Class representing Rasa NLU http server"""
app = Klein()
def __init__(self,
data_router,
loglevel='INFO',
logfile=None,
num_threads=1,
token=None,
cors_origins=None,
testing=False,
default_config_path=None):
self._configure_logging(loglevel, logfile)
self.default_model_config = self._load_default_config(
default_config_path)
self.data_router = data_router
self._testing = testing
self.cors_origins = cors_origins if cors_origins else ["*"]
self.access_token = token
reactor.suggestThreadPoolSize(num_threads * 5)
@staticmethod
def _load_default_config(path):
if path:
return config.load(path).as_dict()
else:
return {}
@staticmethod
def _configure_logging(loglevel, logfile):
logging.basicConfig(filename=logfile, level=loglevel)
logging.captureWarnings(True)
@app.route("/", methods=['GET', 'OPTIONS'])
@check_cors
def hello(self, request):
"""Main Rasa route to check if the server is online"""
return "hello from Rasa NLU: " + __version__
@app.route("/parse", methods=['GET', 'POST', 'OPTIONS'])
@requires_auth
@check_cors
@inlineCallbacks
def parse(self, request):
request.setHeader('Content-Type', 'application/json')
if request.method.decode('utf-8', 'strict') == 'GET':
request_params = decode_parameters(request)
else:
request_params = simplejson.loads(request.content.read().decode(
'utf-8', 'strict'))
if 'query' in request_params:
request_params['q'] = request_params.pop('query')
if 'q' not in request_params:
request.setResponseCode(404)
dumped = json_to_string(
{"error": "Invalid parse parameter specified"})
returnValue(dumped)
else:
data = self.data_router.extract(request_params)
try:
request.setResponseCode(200)
response = yield (self.data_router.parse(data)
if self._testing else threads.deferToThread(
self.data_router.parse, data))
returnValue(json_to_string(response))
except InvalidProjectError as e:
request.setResponseCode(404)
returnValue(json_to_string({"error": "{}".format(e)}))
except Exception as e:
request.setResponseCode(500)
logger.exception(e)
returnValue(json_to_string({"error": "{}".format(e)}))
@app.route("/version", methods=['GET', 'OPTIONS'])
@requires_auth
@check_cors
def version(self, request):
"""Returns the Rasa server's version"""
request.setHeader('Content-Type', 'application/json')
return json_to_string({'version': __version__})
@app.route("/config", methods=['GET', 'OPTIONS'])
@requires_auth
@check_cors
def rasaconfig(self, request):
"""Returns the in-memory configuration of the Rasa server"""
# DEPRECATED: I don't think there is a use case for this endpoint
# anymore - when training a new model, the user should always post
# the configuration as part of the request instead of relying on
# the servers config.
request.setHeader('Content-Type', 'application/json')
return json_to_string(self.default_model_config)
@app.route("/status", methods=['GET', 'OPTIONS'])
@requires_auth
@check_cors
def status(self, request):
request.setHeader('Content-Type', 'application/json')
return json_to_string(self.data_router.get_status())
@app.route("/train", methods=['POST', 'OPTIONS'])
@requires_auth
@check_cors
@inlineCallbacks
def train(self, request):
# if not set will use the default project name, e.g. "default"
project = parameter_or_default(request, "project", default=None)
# if set will not generate a model name but use the passed one
model_name = parameter_or_default(request, "model", default=None)
request_content = request.content.read().decode('utf-8', 'strict')
if is_yaml_request(request):
# assumes the user submitted a model configuration with a data
# parameter attached to it
model_config = utils.read_yaml(request_content)
data = model_config.get("data")
else:
# assumes the caller just provided training data without config
# this will use the default model config the server
# was started with
model_config = self.default_model_config
data = request_content
data_file = dump_to_data_file(data)
request.setHeader('Content-Type', 'application/json')
try:
request.setResponseCode(200)
response = yield self.data_router.start_train_process(
data_file, project, RasaNLUModelConfig(model_config),
model_name)
returnValue(
json_to_string(
{'info': 'new model trained: {}'
''.format(response)}))
except AlreadyTrainingError as e:
request.setResponseCode(403)
returnValue(json_to_string({"error": "{}".format(e)}))
except InvalidProjectError as e:
request.setResponseCode(404)
returnValue(json_to_string({"error": "{}".format(e)}))
except TrainingException as e:
request.setResponseCode(500)
returnValue(json_to_string({"error": "{}".format(e)}))
@app.route("/evaluate", methods=['POST', 'OPTIONS'])
@requires_auth
@check_cors
@inlineCallbacks
def evaluate(self, request):
data_string = request.content.read().decode('utf-8', 'strict')
params = {
key.decode('utf-8', 'strict'): value[0].decode('utf-8', 'strict')
for key, value in request.args.items()
}
request.setHeader('Content-Type', 'application/json')
try:
request.setResponseCode(200)
response = yield self.data_router.evaluate(data_string,
params.get('project'),
params.get('model'))
returnValue(json_to_string(response))
except Exception as e:
request.setResponseCode(500)
returnValue(json_to_string({"error": "{}".format(e)}))
@app.route("/models", methods=['DELETE', 'OPTIONS'])
@requires_auth
@check_cors
def unload_model(self, request):
params = {
key.decode('utf-8', 'strict'): value[0].decode('utf-8', 'strict')
for key, value in request.args.items()
}
request.setHeader('Content-Type', 'application/json')
try:
request.setResponseCode(200)
response = self.data_router.unload_model(
params.get('project', RasaNLUModelConfig.DEFAULT_PROJECT_NAME),
params.get('model'))
return simplejson.dumps(response)
except Exception as e:
request.setResponseCode(500)
logger.exception(e)
return simplejson.dumps({"error": "{}".format(e)})
def test_example(self):
"""
When the endpoint being documented references an example HTTP session
that session is included in the generated API documentation, marked up
as HTTP.
"""
app = Klein()
@app.route("/", methods=["GET"])
@user_documentation("""
Demonstrates examples.
""", ["example-example"])
def hasExamples():
pass
examples = {
u"example-example": {
u"id": u"example-example",
u"doc": u"This example demonstrates examples.",
u"request": u"GET /prefix/ HTTP/1.1\n"
u"\n",
u"response": u"HTTP/1.1 200 OK\n"
u"\n"
u'"%(DOMAIN)s"\n',
},
}
rest = list(makeRst(b"/prefix", app, examples.get, {}))
self.assertEqual(
[
'',
# This line introduces the endpoint
'.. http:get:: /prefix/',
'',
# Here is the prose documentation for the endpoint.
' Demonstrates examples.',
' ',
' **Example:** This example demonstrates examples.',
' ',
# This is a header introducing the request portion of the session.
' Request',
# This blank line is necessary to satisfy reST for some reason.
' ',
' .. sourcecode:: http',
' ',
# Here's the bytes of the HTTP request.
' GET /prefix/ HTTP/1.1',
' Host: api.example.com',
' Content-Type: application/json',
' ',
# This blank line is necessary to satisfy reST for some reason.
' ',
# The same again but for the HTTP response.
' Response',
' ',
' .. sourcecode:: http',
' ',
' HTTP/1.1 200 OK',
' Content-Type: application/json',
' ',
' "example.com"',
' ',
'',
],
rest)
class Server:
app = Klein()
def __init__(self, model_directory, interpreter):
self.model_directory = model_directory
self.interpreter = interpreter
self.agent = self._create_agent(model_directory, interpreter)
@staticmethod
def _create_agent(model_directory, interpreter):
"""Creates a Rasa Agent which runs when the server is started"""
try:
endpoints = AvailableEndpoints.read_endpoints('endpoints.yml')
return Agent.load(model_directory, interpreter, action_endpoint=endpoints.action)
except Exception as e:
logger.warn("Failed to load any agent model. Running "
"Rasa Core server with out loaded model now. {}"
"".format(e))
return None
@app.route("/api/v1/status", methods=['GET'])
def status(self, request):
"""Check if the server is running and responds with the status."""
request.setHeader('Access-Control-Allow-Origin', '*')
return json.dumps({'status': 'OK'})
@app.route('/api/v1/<sender_id>/parse', methods=['GET', 'POST'])
def parse(self, request, sender_id):
request.setHeader('Content-Type', 'application/json')
request_params = request_parameters(request)
if 'query' in request_params:
message = request_params.pop('query')
elif 'q' in request_params:
message = request_params.pop('q')
else:
request.setResponseCode(400)
return json.dumps({"error": "Invalid parse parameter specified"})
try:
response = self.agent.handle_message(message, sender_id)
request.setResponseCode(200)
return json.dumps(response)
except Exception as e:
request.setResponseCode(500)
logger.error("Caught an exception during "
"parse: {}".format(e), exc_info=1)
return json.dumps({"error": "{}".format(e)})
@app.route('/api/v1/<sender_id>/respond', methods=['GET', 'POST'])
def respond(self, request, sender_id):
request.setHeader('Content-Type', 'application/json')
request.setHeader('Access-Control-Allow-Origin', '*')
request_params = request_parameters(request)
if 'query' in request_params:
message = request_params.pop('query')
elif 'q' in request_params:
message = request_params.pop('q')
else:
request.setResponseCode(400)
return json.dumps({"error": "Invalid parse parameter specified"})
try:
out = CollectingOutputChannel()
response = self.agent.handle_text(message, output_channel=out, sender_id=sender_id)
request.setResponseCode(200)
return json.dumps(response)
except Exception as e:
request.setResponseCode(500)
logger.error("Caught an exception during "
"parse: {}".format(e), exc_info=1)
return json.dumps({"error": "{}".format(e)})
Y_pred_text = text_pred
# construct output, highlight diacritics
rez = []
for ch0, ch2 in zip(text0, Y_pred_text):
ch = ch0
if ch0.lower() != ch2.lower() and ch2 != " ":
ch = "<span class='mod'>" + ch2 + "</span>"
rez.append(ch)
rez_str = "".join(rez).strip()
return rez_str
#DBG()
app = Klein()
@app.route("/ajax")
def generate_ajax(request):
txt = request.content.read().decode("utf-8")
print("GOT TXT=", txt, type(txt))
request.setHeader('Content-Type', 'text/html; charset=utf-8')
request.write(predict(model, txt).encode("utf-8"))
@app.route("/", branch=True)
def generate_index(request):
return File("./app")
class RestApi:
app = Klein()
notif = None
def __init__(self):
self.notif = NotificationDB.instance()
#
# REST API Routes
#
@app.route('/')
def home(self, request):
endpoints_html = """<ul>
<li><pre>{apiPrefix}/notifications/block/<height></pre> <em>notifications by block</em></li>
<li><pre>{apiPrefix}/notifications/addr/<addr></pre><em>notifications by address</em></li>
<li><pre>{apiPrefix}/notifications/tx/<hash></pre><em>notifications by tx</em></li>
<li><pre>{apiPrefix}/notifications/contract/<hash></pre><em>notifications by contract</em></li>
<li><pre>{apiPrefix}/tokens</pre><em>lists all NEP5 Tokens</em></li>
<li><pre>{apiPrefix}/token/<contract_hash></pre><em>list an NEP5 Token</em></li>
<li><pre>{apiPrefix}/status</pre> <em>current block height and version</em></li>
</ul>
""".format(apiPrefix=API_URL_PREFIX)
return """<html>
<style>body {padding:20px;max-width:800px;pre { background-color:#eee; }</style>
<body>
<p>
<h2>REST API for NEO %s</h2>
(see also <a href="https://github.com/CityOfZion/neo-python">neo-python</a>, <a href="https://github.com/CityOfZion/neo-python/blob/development/api-server.py">api-server.py</a>)
</p>
<hr/>
<h2>endpoints:</h2>
<p>%s</p>
<div>
<hr/>
<h3>pagination</h3>
<p>results are offered in page size of 500</p>
<p>you may request a different page by specifying the <code>page</code> query string param, for example:</p>
<pre>/block/123456?page=3</pre>
<p>page index starts at 0, so the 2nd page would be <code>?page=1</code></p>
<hr/>
<h3>sample output</h3>
<pre>
{
"page_len": 1000,
"total_pages": 1,
"total": 4,
"page": 1,
"current_height": 982506,
"results": [
{
"type": "SmartContract.Runtime.Notify",
"contract": "400cbed5b41014788d939eaf6286e336e7140f8c",
"tx": "d0805fd7ec19a4a414374ae3720447d2576659053eb7588b85a0f9f1fd629791",
"block": 928119,
"addr_from": "AUYSKFEWPZxP57fo3TsK6Lwg22qxSFupKF",
"amount": 1,
"addr_to": "ALULT5WpeiHnEXYFe72Yq7nRB3ZBmsBypq",
"notify_type": "transfer"
},
{
"type": "SmartContract.Runtime.Notify",
"contract": "d3de84c166d93ad2581cb587bda8e02b12dc37ca",
"tx": "667df082eaa16ce2b07e48e214eb019b3e9450e76daea4f5b0450578a07836ef",
"block": 936352,
"addr_from": "ALULT5WpeiHnEXYFe72Yq7nRB3ZBmsBypq",
"amount": 1,
"addr_to": "AaD74SkQXsSE7UtSutQ4VV3mRdQUoMk98X",
"notify_type": "transfer"
},
{
"type": "SmartContract.Runtime.Notify",
"contract": "2c0fdfa9592814b0a938219e218e3a6b08615acd",
"tx": "eda792e7814e128eecda992f78a11577ee0604827de4aa91ffcda4616c889191",
"block": 939449,
"addr_from": "ALULT5WpeiHnEXYFe72Yq7nRB3ZBmsBypq",
"amount": 1,
"addr_to": "AaVgSU9vEPdwc49rPrCyj1LYkpsGFNgbjy",
"notify_type": "transfer"
},
{
"type": "SmartContract.Runtime.Notify",
"contract": "f9572c5b119a6b5775a6af07f1cef5d310038f55",
"tx": "6d0f1decbf3874d08d41f2cc9e8672cd3507c962668c15793e3dd3e01fc3551c",
"block": 942369,
"addr_from": "ALULT5WpeiHnEXYFe72Yq7nRB3ZBmsBypq",
"amount": 1,
"addr_to": "APaGQT4dx4gUDApVPnbtZvChJ8UKRsZBdt",
"notify_type": "transfer"
}
],
"message": ""
}</pre>
</div>
</body>
</html>""" % (settings.net_name, endpoints_html)
@app.route('%s/notifications/block/<int:block>' % API_URL_PREFIX,
methods=['GET'])
@cors_header
def get_by_block(self, request, block):
request.setHeader('Content-Type', 'application/json')
try:
if int(block) > Blockchain.Default().Height:
return self.format_message("Higher than current block")
else:
notifications = self.notif.get_by_block(block)
except Exception as e:
logger.info("Could not get notifications for block %s %s" %
(block, e))
return self.format_message(
"Could not get notifications for block %s because %s " %
(block, e))
return self.format_notifications(request, notifications)
@app.route('%s/notifications/addr/<string:address>' % API_URL_PREFIX,
methods=['GET'])
@cors_header
def get_by_addr(self, request, address):
request.setHeader('Content-Type', 'application/json')
try:
notifications = self.notif.get_by_addr(address)
except Exception as e:
logger.info("Could not get notifications for address %s " %
address)
return self.format_message(
"Could not get notifications for address %s because %s" %
(address, e))
return self.format_notifications(request, notifications)
@app.route('%s/notifications/tx/<string:tx_hash>' % API_URL_PREFIX,
methods=['GET'])
@cors_header
def get_by_tx(self, request, tx_hash):
request.setHeader('Content-Type', 'application/json')
bc = Blockchain.Default() # type: Blockchain
notifications = []
try:
hash = UInt256.ParseString(tx_hash)
tx, height = bc.GetTransaction(hash)
if not tx:
return self.format_message(
"Could not find transaction for hash %s" % (tx_hash))
block_notifications = self.notif.get_by_block(height - 1)
for n in block_notifications:
if n.tx_hash == tx.Hash:
notifications.append(n)
except Exception as e:
logger.info("Could not get tx with hash %s because %s " %
(tx_hash, e))
return self.format_message(
"Could not get tx with hash %s because %s " % (tx_hash, e))
return self.format_notifications(request, notifications)
@app.route('%s/notifications/contract/<string:contract_hash>' %
API_URL_PREFIX,
methods=['GET'])
@cors_header
def get_by_contract(self, request, contract_hash):
request.setHeader('Content-Type', 'application/json')
try:
hash = UInt160.ParseString(contract_hash)
notifications = self.notif.get_by_contract(hash)
except Exception as e:
logger.info("Could not get notifications for contract %s " %
contract_hash)
return self.format_message(
"Could not get notifications for contract hash %s because %s" %
(contract_hash, e))
return self.format_notifications(request, notifications)
@app.route('%s/tokens' % API_URL_PREFIX, methods=['GET'])
@cors_header
def get_tokens(self, request):
request.setHeader('Content-Type', 'application/json')
notifications = self.notif.get_tokens()
return self.format_notifications(request, notifications)
@app.route('%s/token/<string:contract_hash>' % API_URL_PREFIX,
methods=['GET'])
@cors_header
def get_token(self, request, contract_hash):
request.setHeader('Content-Type', 'application/json')
try:
uint160 = UInt160.ParseString(contract_hash)
contract_event = self.notif.get_token(uint160)
if not contract_event:
return self.format_message(
"Could not find contract with hash %s" % contract_hash)
notifications = [contract_event]
except Exception as e:
logger.info("Could not get contract with hash %s because %s " %
(contract_hash, e))
return self.format_message(
"Could not get contract with hash %s because %s " %
(contract_hash, e))
return self.format_notifications(request, notifications)
@app.route('%s/status' % API_URL_PREFIX, methods=['GET'])
@cors_header
def get_status(self, request):
request.setHeader('Content-Type', 'application/json')
return json.dumps(
{
'current_height': Blockchain.Default().Height + 1,
'version': settings.VERSION_NAME,
'num_peers': len(NodeLeader.Instance().Peers)
},
indent=4,
sort_keys=True)
def format_notifications(self, request, notifications, show_none=False):
notif_len = len(notifications)
page_len = 500
page = 1
message = ''
if b'page' in request.args:
try:
page = int(request.args[b'page'][0])
except Exception as e:
print("could not get page: %s" % e)
if b'pagesize' in request.args:
try:
page_len = int(request.args[b'pagesize'][0])
except Exception as e:
print("could not get page length: %s" % e)
# note, we want pages to start from 1, not 0, to be
# in sync with C# version
# we'll also convert page 0 to page1
if page == 0:
page = 1
start = page_len * (page - 1)
end = start + page_len
if start > notif_len:
message = 'page greater than result length'
notifications = notifications[start:end]
total_pages = math.ceil(notif_len / page_len)
return json.dumps(
{
'current_height':
Blockchain.Default().Height + 1,
'message':
message,
'total':
notif_len,
'results':
None if show_none else [n.ToJson() for n in notifications],
'page':
page,
'page_len':
page_len,
'total_pages':
total_pages
},
indent=4,
sort_keys=True)
def format_message(self, message):
return json.dumps(
{
'current_height': Blockchain.Default().Height + 1,
'message': message,
'total': 0,
'results': None,
'page': 0,
'page_len': 0,
'total_pages': 0
},
indent=4,
sort_keys=True)
from klein import Klein
app = Klein()
@app.route('/user/<username>')
def pg_user(request, username):
return 'Hi %s!' % (username, )
app.run("localhost", 8080)
class MarathonAcmeServer(object):
app = Klein()
log = Logger()
def __init__(self, responder_resource):
"""
:param responder_resource:
An ``IResponse`` used to respond to ACME HTTP challenge validation
requests.
"""
self.responder_resource = responder_resource
self.health_handler = None
def listen(self, reactor, endpoint_description):
"""
Run the server, i.e. start listening for requests on the given host and
port.
:param reactor: The ``IReactorTCP`` to use.
:param endpoint_description:
The Twisted description for the endpoint to listen on.
:return:
A deferred that returns an object that provides ``IListeningPort``.
"""
endpoint = serverFromString(reactor, endpoint_description)
return endpoint.listen(Site(self.app.resource()))
@app.route('/.well-known/acme-challenge/', branch=True, methods=['GET'])
def acme_challenge(self, request):
"""
Respond to ACME challenge validation requests on
``/.well-known/acme-challenge/`` using the ACME responder resource.
"""
return self.responder_resource
def set_health_handler(self, health_handler):
"""
Set the handler for the health endpoint.
:param health_handler:
The handler for health status requests. This must be a callable
that returns a Health object.
"""
self.health_handler = health_handler
@app.route('/health', methods=['GET'])
def health(self, request):
""" Listens to incoming health checks from Marathon on ``/health``. """
if self.health_handler is None:
return self._no_health_handler(request)
health = self.health_handler()
response_code = OK if health.healthy else SERVICE_UNAVAILABLE
request.setResponseCode(response_code)
write_request_json(request, health.json_message)
def _no_health_handler(self, request):
self.log.warn('Request to /health made but no handler is set')
request.setResponseCode(NOT_IMPLEMENTED)
write_request_json(
request,
{'error': 'Cannot determine service health: no handler set'})
if sys.version_info[0] == 3:
xrange = range
DBDRIVER = 'mysql'
DBHOSTNAME = os.environ.get('DBHOST', 'localhost')
DATABASE_URI = '%s://benchmarkdbuser:benchmarkdbpass@%s:3306/hello_world?charset=utf8' % (DBDRIVER, DBHOSTNAME)
Base = declarative_base()
db_engine = create_engine(DATABASE_URI)
Session = sessionmaker(bind=db_engine)
db_session = Session()
env = Environment(loader=PackageLoader("app", "templates"), autoescape=True, auto_reload=False)
app = Klein()
class Fortune(Base):
__tablename__ = "Fortune"
id = Column(Integer, primary_key=True)
message = Column(String)
def serialize(self):
return {
'id': self.id,
'randomNumber': self.randomNumber,
}
class World(Base):
__tablename__ = "World"
def test_inputSchema(self):
"""
The generated API documentation includes the input schema.
"""
app = Klein()
@app.route(b"/", methods=[b"GET"])
@structured(
inputSchema={'$ref': '/v0/test.json#/endpoint'},
outputSchema={},
schema_store=self.INPUT_SCHEMAS,
)
def f():
"""
Developer docs,
"""
rest = list(makeRst(b"/prefix", app, None, self.INPUT_SCHEMAS))
self.assertEqual(
rest,
[
'',
'.. http:get:: /prefix/',
'',
' Undocumented.',
' ',
' .. hidden-code-block:: json',
' :label: + Request JSON Schema',
' :starthidden: True',
' ',
' {',
' "$schema": "http://json-schema.org/draft-04/schema#",',
' "properties": {',
' "optional": {',
' "description": "one\\ntwo",',
' "title": "TITLE",',
' "type": "string"',
' },',
' "param": {',
' "description": "one\\ntwo",',
' "title": "TITLE",',
' "type": "string"',
' }',
' },',
' "required": [',
' "param"',
' ],',
' "type": "object"',
' }',
' ',
# YAML is unorderd :(
' :<json string optional: TITLE',
' ',
' one',
' two',
' ',
' :<json string param: *(required)* TITLE',
' ',
' one',
' two',
' ',
'',
])
api,
"execute",
capability_version,
capability_hash, '')
links.append({"href": capability_url, "rel": "capability"})
return links
else:
return url
def transaction_id(request):
"""
Extract the transaction id from the given request.
:param IRequest request: The request we are trying to get the
transaction id for.
:returns: A string transaction id.
"""
return request.responseHeaders.getRawHeaders('X-Response-Id')[0]
app = Klein()
app.route = partial(app.route, strict_slashes=False)
root = Resource()
root.putChild('v1.0', app.resource())
root.putChild('', Data('', 'text/plain'))
class ConfigApi(object):
"""
Configuration API.
This class handles HTTP API calls related to router configuration.
"""
routes = Klein()
def __init__(self, update_manager, update_fetcher):
self.update_manager = update_manager
self.update_fetcher = update_fetcher
@routes.route('/hostconfig', methods=['PUT'])
def update_hostconfig(self, request):
"""
Replace the device's host configuration.
**Example request**:
.. sourcecode:: http
PUT /api/v1/config/hostconfig
Content-Type: application/json
{
"firewall": {
"defaults": {
"forward": "ACCEPT",
"input": "ACCEPT",
"output": "ACCEPT"
}
},
...
}
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Type: application/json
{
change_id: 1
}
For a complete example, please see the Host Configuration section.
"""
cors.config_cors(request)
body = str2json(request.content.read())
config = body['config']
update = dict(updateClass='ROUTER',
updateType='sethostconfig',
name=constants.RESERVED_CHUTE,
tok=timeint(),
hostconfig=config)
# We will return the change ID to the caller for tracking and log
# retrieval.
update['change_id'] = self.update_manager.assign_change_id()
self.update_manager.add_update(**update)
result = {'change_id': update['change_id']}
request.setHeader('Content-Type', 'application/json')
return json.dumps(result)
@routes.route('/hostconfig', methods=['GET'])
def get_hostconfig(self, request):
"""
Get the device's current host configuration.
**Example request**:
.. sourcecode:: http
GET /api/v1/config/hostconfig
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Type: application/json
{
"firewall": {
"defaults": {
"forward": "ACCEPT",
"input": "ACCEPT",
"output": "ACCEPT"
}
},
...
}
For a complete example, please see the Host Configuration section.
"""
cors.config_cors(request)
config = hostconfig.prepareHostConfig()
request.setHeader('Content-Type', 'application/json')
return json.dumps(config, separators=(',', ':'))
@routes.route('/new-config', methods=['GET'])
def new_config(self, request):
"""
Generate a new node configuration based on the hardware.
**Example request**:
.. sourcecode:: http
GET /api/v1/config/new_config
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Type: application/json
{
"firewall": {
"defaults": {
"forward": "ACCEPT",
"input": "ACCEPT",
"output": "ACCEPT"
}
},
...
}
For a complete example, please see the Host Configuration section.
"""
cors.config_cors(request)
config = hostconfig.prepareHostConfig(hostConfigPath='/dev/null')
request.setHeader('Content-Type', 'application/json')
return json.dumps(config, separators=(',', ':'))
@routes.route('/pdid', methods=['GET'])
def get_pdid(self, request):
"""
Get the device's current ParaDrop ID. This is the identifier assigned
by the cloud controller.
**Example request**:
.. sourcecode:: http
GET /api/v1/config/pdid
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Type: application/json
{
pdid: "5890e1e5ab7e317e6c6e049f"
}
"""
cors.config_cors(request)
pdid = nexus.core.info.pdid
if pdid is None:
pdid = ""
request.setHeader('Content-Type', 'application/json')
return json.dumps({'pdid': pdid})
@routes.route('/provision', methods=['POST'])
def provision(self, request):
"""
Provision the device with credentials from a cloud controller.
"""
cors.config_cors(request)
body = str2json(request.content.read())
routerId = body['routerId']
apitoken = body['apitoken']
pdserver = body['pdserver']
wampRouter = body['wampRouter']
changed = False
if routerId != nexus.core.info.pdid \
or pdserver != nexus.core.info.pdserver \
or wampRouter != nexus.core.info.wampRouter:
if pdserver and wampRouter:
nexus.core.provision(routerId, pdserver, wampRouter)
else:
nexus.core.provision(routerId)
changed = True
if apitoken != nexus.core.getKey('apitoken'):
nexus.core.saveKey(apitoken, 'apitoken')
changed = True
if changed:
PDServerRequest.resetToken()
nexus.core.jwt_valid = False
def set_update_fetcher(session):
session.set_update_fetcher(self.update_fetcher)
@inlineCallbacks
def start_polling(result):
yield self.update_fetcher.start_polling()
def send_response(result):
response = dict()
response['provisioned'] = True
response['httpConnected'] = nexus.core.jwt_valid
response['wampConnected'] = nexus.core.wamp_connected
request.setHeader('Content-Type', 'application/json')
return json.dumps(response)
wampDeferred = nexus.core.connect(WampSession)
wampDeferred.addCallback(set_update_fetcher)
httpDeferred = sendStateReport()
httpDeferred.addCallback(start_polling)
identDeferred = sendNodeIdentity()
dl = DeferredList([wampDeferred, httpDeferred, identDeferred],
consumeErrors=True)
dl.addBoth(send_response)
reactor.callLater(6, dl.cancel)
return dl
else:
return json.dumps({
'success':
False,
'message':
'No change on the provision parameters'
})
@routes.route('/provision', methods=['GET'])
def get_provision(self, request):
"""
Get the provision status of the device.
"""
cors.config_cors(request)
result = dict()
result['routerId'] = nexus.core.info.pdid
result['pdserver'] = nexus.core.info.pdserver
result['wampRouter'] = nexus.core.info.wampRouter
apitoken = nexus.core.getKey('apitoken')
result['provisioned'] = (result['routerId'] is not None and \
apitoken is not None)
result['httpConnected'] = nexus.core.jwt_valid
result['wampConnected'] = nexus.core.wamp_connected
request.setHeader('Content-Type', 'application/json')
return json.dumps(result)
@routes.route('/settings', methods=['GET'])
def get_settings(self, request):
"""
Get current values of system settings.
These are the values from paradrop.base.settings. Settings are loaded
at system initialization from the settings.ini file and environment
variables. They are intended to be read-only after initialization.
This endpoint returns the settings as a dictionary with lowercase field
names.
Example:
{
"portal_server_port": 8080,
...
}
"""
cors.config_cors(request)
request.setHeader('Content-Type', 'application/json')
result = {}
for name, value in settings.iterate_module_attributes(settings):
result[name.lower()] = value
return json.dumps(result)
@routes.route('/startUpdate', methods=['POST'])
def start_update(self, request):
cors.config_cors(request)
self.update_manager.startUpdate()
request.setHeader('Content-Type', 'application/json')
return json.dumps({'success': True})
@routes.route('/factoryReset', methods=['POST'])
@inlineCallbacks
def factory_reset(self, request):
"""
Initiate the factory reset process.
"""
cors.config_cors(request)
update = dict(updateClass='ROUTER',
updateType='factoryreset',
name='PARADROP',
tok=timeint())
update = yield self.update_manager.add_update(**update)
returnValue(json.dumps(update.result))
@routes.route('/pdconf', methods=['GET'])
def pdconf(self, request):
"""
Get configuration sections from pdconf.
This returns a list of configuration sections and whether they were
successfully applied. This is intended for debugging purposes.
"""
cors.config_cors(request)
request.setHeader('Content-Type', 'application/json')
return pdconf_client.systemStatus()
@routes.route('/pdconf', methods=['PUT'])
def pdconf_reload(self, request):
"""
Trigger pdconf to reload UCI configuration files.
Trigger pdconf to reload UCI configuration files and return the status.
This function is intended for low-level debugging of the paradrop
pdconf module.
"""
cors.config_cors(request)
request.setHeader('Content-Type', 'application/json')
return pdconf_client.reloadAll()
@routes.route('/sshKeys/<user>', methods=['GET', 'POST'])
def sshKeys(self, request, user):
"""
Manage list of authorized keys for SSH access.
"""
cors.config_cors(request)
request.setHeader('Content-Type', 'application/json')
if request.method == "GET":
try:
keys = ssh_keys.getAuthorizedKeys(user)
return json.dumps(keys)
except Exception as e:
out.warn(str(e))
request.setResponseCode(404)
return json.dumps({'message': str(e)})
else:
body = str2json(request.content.read())
key = body['key'].strip()
try:
ssh_keys.addAuthorizedKey(key, user)
return json.dumps(body)
except Exception as e:
out.warn(str(e))
request.setResponseCode(404)
return json.dumps({'message': str(e)})
class MyWebMail(object):
app = Klein()
buckets = {}
def __init__(self):
self.init_stuff()
def init_stuff(self):
cli = riak.RiakClient()
self.buckets['cred'] = cli.bucket('cred')
self.buckets['mailmeta'] = cli.bucket('mailmeta')
self.buckets['mailbody'] = cli.bucket('mailbody')
@inlineCallbacks
def getcred(self, key):
obj = yield self.buckets['cred'].get(key)
returnValue(obj.get_data())
# user mgmt here.
@app.route("/profile", methods=['GET'])
@inlineCallbacks
def modifyuser(self, request):
login = request.getUser()
curdata = yield self.getcred(login)
mod_dsl = dsl.replace('NewEmailUser', 'ModifyUser')
d = curdata
d.update({'login': login})
myform = myhtml.renderForm(myhtml.parse(mod_dsl),
action="/profile",
method="POST")
header = '<html><head><title>new user</title>' + '<style type="text/css" media="screen">' + mystyle2 + '</style></head>'
request.write(header +
'<body><h3>Modify User</h3><div id="container">')
form2 = str(myform).replace('NONE', '').replace(
'<label>Login',
'<fieldset><legend>User info</legend><div class="fm-req"><label>Login'
)
form2 = form2.replace('</form>', '</div></div></form>')
tmp = str(form2) % d
request.write(tmp.encode('ascii'))
request.write('</body></html>')
@app.route("/profile", methods=['POST'])
@inlineCallbacks
def modifyuser_post(self, request):
login = request.getUser()
a = request.args
data = yield self.getcred(login)
name = a.get('name')[0]
password = a.get('password')[0]
# check!
if len(password) < 8:
request.write(
'<b>Password was too short! need min. 8 characters.</b>')
else:
data.update({
"name": name,
"password": password,
"timestamp": int(time.time())
})
obj = self.buckets['cred'].new(login, data)
yield obj.store()
print('saved modified entry to users!')
request.write('Modified user %s. ' % (login))
request.write('<p><a href="/profile">Return.</a>')
# webmail routes here.
@app.route('/')
def index(self, request):
return 'it works...'
@app.route('/search', methods=['GET'])
def search(self, request):
out = '''
<h2>full text search</h2>
<p>
type your search term(s) here:
<form action="/search" method="POST" >
<input type="TEXT" name="terms" value="hae..." />
<input type="SUBMIT" name="SUBMIT" />
</form>
'''
request.write(out)
@app.route('/search', methods=['POST'])
@inlineCallbacks
def do_search(self, request):
import quopri, re
a = request.args
login = request.getUser()
obj = yield self.buckets['mailmeta'].get(login)
user_emails = obj.get_data().keys()
terms = a.get('terms')[0]
print 'TERMS:', terms
res = yield self.buckets['mailbody'].search(terms)
request.write('search found this:')
r1 = re.compile(r".*Delivered-To:\s([a-zA-Z0-9.-]+)@.*?\n.*",
re.DOTALL) # is this all?
for doc in res['docs']:
data = doc['value']
m = r1.match(data)
if m.group(1) != login:
continue # this msg was not for this user, skip it.
content_trans = 'Content-Transfer-Encoding: quoted-printable'
if content_trans in data:
headers, _, body = data.partition(content_trans)
else:
headers, body = data, ''
out = headers.decode('ascii') # headers should always be ascii-onl
s = quopri.decodestring(body).decode('utf-8', errors='replace')
print type(out), type(s)
out = out + s
out = out.encode('utf-8')
request.write('<pre>' + out + '</pre>')
@app.route('/showSess')
def showSess(self, request):
s = request.getSession()
print request.getUser()
print 'DEBUG:', s.sessionNamespaces, dir(s.sessionNamespaces)
return 'Session id is: ' + s.uid
@app.route('/expireSess')
def expSess(self, request):
request.getSession().expire()
return 'Session expired.'
@app.route('/inbox')
@defer.inlineCallbacks
def inbox(self, request):
user = request.getUser()
obj = yield self.buckets['mailmeta'].get(user)
data = obj.get_data()
print 'DEBUG:', data, type(data)
out1 = 'Num of mails: %d' % len(data)
out2 = 'Total bytes used: %d' % sum([data[x]['size'] for x in data])
request.write('<p>'.join([out1, out2]).encode('ascii'))
# list all mail headers
for mailid in data:
request.write('<p>')
obj = yield self.buckets['mailbody'].get(mailid)
body = obj.get_data()
#print body, type(body)
lst = body.split('\n')
out = []
for line in lst:
if line.startswith('From:') or line.startswith(
'To:') or line.startswith('Subject:'):
out.append(line)
request.write('<br>'.join(out))
class ChuteApi(object):
routes = Klein()
def __init__(self, update_manager):
self.update_manager = update_manager
@routes.route('/', methods=['GET'])
def get_chutes(self, request):
"""
List installed chutes.
**Example request**:
.. sourcecode:: http
GET /api/v1/chutes/
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Type: application/json
[
{
"environment": {},
"name": "hello-world",
"allocation": {
"cpu_shares": 1024,
"prioritize_traffic": false
},
"state": "running",
"version": "x1511808778",
"resources": null
}
]
"""
cors.config_cors(request)
request.setHeader('Content-Type', 'application/json')
chuteStorage = ChuteStorage()
chutes = chuteStorage.getChuteList()
allocation = resource.computeResourceAllocation(chutes)
result = []
for chute in chutes:
container = ChuteContainer(chute.name)
result.append({
'name': chute.name,
'state': container.getStatus(),
'version': getattr(chute, 'version', None),
'allocation': allocation.get(chute.name, None),
'environment': getattr(chute, 'environment', None),
'resources': getattr(chute, 'resources', None)
})
return json.dumps(result)
@routes.route('/', methods=['POST'])
def create_chute(self, request):
cors.config_cors(request)
update = dict(updateClass='CHUTE',
updateType='create',
tok=pdutils.timeint())
ctype = request.requestHeaders.getRawHeaders('Content-Type',
default=[None])[0]
if ctype == "application/x-tar":
workdir, paradrop_yaml = extract_tarred_chute(request.content)
config = paradrop_yaml.get("config", {})
# Try to read chute name from top level (preferred) or from config
# object (deprecated).
if 'name' in paradrop_yaml:
update['name'] = paradrop_yaml['name']
elif 'name' in config:
out.warn(
"Deprecated: move chute name to top level of config file.")
update['name'] = config['name']
else:
raise Exception("Chute name not found in configuration file.")
update['workdir'] = workdir
chute_version = paradrop_yaml.get("version", None)
update['version'] = "x{}".format(update['tok'])
update.update(config)
else:
# TODO: this case is not tested
body = json.loads(request.content.read())
config = body['config']
update.update(config)
# Set a time-based version number for side-loaded chutes because we do
# not expect they to receive it from the config file.
update['version'] = "x{}".format(update['tok'])
# We will return the change ID to the caller for tracking and log
# retrieval.
update['change_id'] = self.update_manager.assign_change_id()
d = self.update_manager.add_update(**update)
result = {'change_id': update['change_id']}
request.setHeader('Content-Type', 'application/json')
return json.dumps(result)
@routes.route('/<chute>', methods=['GET'])
def get_chute(self, request, chute):
"""
Get information about an installed chute.
**Example request**:
.. sourcecode:: http
GET /api/v1/chutes/hello-world
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Type: application/json
{
"environment": {},
"name": "hello-world",
"allocation": {
"cpu_shares": 1024,
"prioritize_traffic": false
},
"state": "running",
"version": "x1511808778",
"resources": null
}
"""
cors.config_cors(request)
request.setHeader('Content-Type', 'application/json')
chute_obj = ChuteStorage.chuteList[chute]
container = ChuteContainer(chute)
chuteStorage = ChuteStorage()
allocation = resource.computeResourceAllocation(
chuteStorage.getChuteList())
result = {
'name': chute,
'state': container.getStatus(),
'version': getattr(chute_obj, 'version', None),
'allocation': allocation.get(chute, None),
'environment': getattr(chute_obj, 'environment', None),
'resources': getattr(chute_obj, 'resources', None)
}
return json.dumps(result)
@routes.route('/<chute>', methods=['PUT'])
def update_chute(self, request, chute):
cors.config_cors(request)
update = dict(updateClass='CHUTE',
updateType='update',
tok=pdutils.timeint(),
name=chute)
ctype = request.requestHeaders.getRawHeaders('Content-Type',
default=[None])[0]
if ctype == "application/x-tar":
workdir, paradrop_yaml = extract_tarred_chute(request.content)
config = paradrop_yaml.get("config", {})
# Try to read chute name from top level (preferred) or from config
# object (deprecated).
if 'name' in paradrop_yaml:
update['name'] = paradrop_yaml['name']
elif 'name' in config:
out.warn(
"Deprecated: move chute name to top level of config file.")
update['name'] = config['name']
else:
raise Exception("Chute name not found in configuration file.")
update['workdir'] = workdir
chute_version = paradrop_yaml.get("version", None)
update['version'] = "x{}".format(update['tok'])
update.update(config)
else:
body = json.loads(request.content.read())
config = body['config']
update.update(config)
# Set a time-based version number for side-loaded chutes because we do
# not expect the to receive it from the config file.
update['version'] = "x{}".format(update['tok'])
# We will return the change ID to the caller for tracking and log
# retrieval.
update['change_id'] = self.update_manager.assign_change_id()
d = self.update_manager.add_update(**update)
result = {'change_id': update['change_id']}
request.setHeader('Content-Type', 'application/json')
return json.dumps(result)
@routes.route('/<chute>', methods=['DELETE'])
def delete_chute(self, request, chute):
cors.config_cors(request)
update = dict(updateClass='CHUTE',
updateType='delete',
tok=pdutils.timeint(),
name=chute)
# We will return the change ID to the caller for tracking and log
# retrieval.
update['change_id'] = self.update_manager.assign_change_id()
d = self.update_manager.add_update(**update)
result = {'change_id': update['change_id']}
request.setHeader('Content-Type', 'application/json')
return json.dumps(result)
@routes.route('/<chute>/stop', methods=['POST'])
def stop_chute(self, request, chute):
cors.config_cors(request)
update = dict(updateClass='CHUTE',
updateType='stop',
tok=pdutils.timeint(),
name=chute)
# We will return the change ID to the caller for tracking and log
# retrieval.
update['change_id'] = self.update_manager.assign_change_id()
d = self.update_manager.add_update(**update)
result = {'change_id': update['change_id']}
request.setHeader('Content-Type', 'application/json')
return json.dumps(result)
@routes.route('/<chute>/start', methods=['POST'])
def start_chute(self, request, chute):
cors.config_cors(request)
update = dict(updateClass='CHUTE',
updateType='start',
tok=pdutils.timeint(),
name=chute)
try:
body = json.loads(request.content.read())
# Chute environment variables can be replaced during the operation.
update['environment'] = body['environment']
except Exception as error:
pass
# We will return the change ID to the caller for tracking and log
# retrieval.
update['change_id'] = self.update_manager.assign_change_id()
d = self.update_manager.add_update(**update)
result = {'change_id': update['change_id']}
request.setHeader('Content-Type', 'application/json')
return json.dumps(result)
@routes.route('/<chute>/restart', methods=['POST'])
def restart_chute(self, request, chute):
cors.config_cors(request)
update = dict(updateClass='CHUTE',
updateType='restart',
tok=pdutils.timeint(),
name=chute)
try:
body = json.loads(request.content.read())
# Chute environment variables can be replaced during the operation.
update['environment'] = body['environment']
except Exception as error:
pass
# We will return the change ID to the caller for tracking and log
# retrieval.
update['change_id'] = self.update_manager.assign_change_id()
d = self.update_manager.add_update(**update)
result = {'change_id': update['change_id']}
request.setHeader('Content-Type', 'application/json')
return json.dumps(result)
@routes.route('/<chute>/cache', methods=['GET'])
def get_chute_cache(self, request, chute):
"""
Get chute cache contents.
The chute cache is a key-value store used during chute installation.
It can be useful for debugging the Paradrop platform.
"""
cors.config_cors(request)
request.setHeader('Content-Type', 'application/json')
chute_obj = ChuteStorage.chuteList[chute]
result = chute_obj.getCacheContents()
return json.dumps(result, cls=ChuteCacheEncoder)
@routes.route('/<chute>/config', methods=['GET'])
def get_chute_config(self, request, chute):
"""
Get current chute configuration.
**Example request**:
.. sourcecode:: http
GET /api/v1/chutes/captive-portal/config
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Type: application/json
{
"net": {
"wifi": {
"dhcp": {
"lease": "1h",
"limit": 250,
"start": 3
},
"intfName": "wlan0",
"options": {
"isolate": True
},
"ssid": "Free WiFi",
"type": "wifi"
}
}
}
"""
cors.config_cors(request)
request.setHeader('Content-Type', 'application/json')
chute_obj = ChuteStorage.chuteList[chute]
config = chute_obj.getConfiguration()
return json.dumps(config)
@routes.route('/<chute>/config', methods=['PUT'])
def set_chute_config(self, request, chute):
"""
Update the chute configuration and restart to apply changes.
**Example request**:
.. sourcecode:: http
PUT /api/v1/chutes/captive-portal/config
Content-Type: application/json
{
"net": {
"wifi": {
"dhcp": {
"lease": "1h",
"limit": 250,
"start": 3
},
"intfName": "wlan0",
"options": {
"isolate": True
},
"ssid": "Better Free WiFi",
"type": "wifi"
}
}
}
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Type: application/json
{
"change_id": 1
}
"""
cors.config_cors(request)
request.setHeader('Content-Type', 'application/json')
update = dict(updateClass='CHUTE',
updateType='restart',
tok=pdutils.timeint(),
name=chute)
try:
body = json.loads(request.content.read())
update.update(body)
except Exception as error:
pass
# We will return the change ID to the caller for tracking and log
# retrieval.
update['change_id'] = self.update_manager.assign_change_id()
d = self.update_manager.add_update(**update)
result = {'change_id': update['change_id']}
return json.dumps(result)
@routes.route('/<chute>/networks', methods=['GET'])
def get_networks(self, request, chute):
"""
Get list of networks configured for the chute.
**Example request**:
.. sourcecode:: http
GET /api/v1/chutes/captive-portal/networks
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Type: application/json
[
{
"interface": "wlan0",
"type": "wifi",
"name": "wifi"
}
]
"""
cors.config_cors(request)
request.setHeader('Content-Type', 'application/json')
chute_obj = ChuteStorage.chuteList[chute]
networkInterfaces = chute_obj.getCache('networkInterfaces')
result = []
for iface in networkInterfaces:
data = {
'name': iface['name'],
'type': iface['netType'],
'interface': iface['internalIntf']
}
result.append(data)
return json.dumps(result)
@routes.route('/<chute>/networks/<network>', methods=['GET'])
def get_network(self, request, chute, network):
"""
Get information about a network configured for the chute.
**Example request**:
.. sourcecode:: http
GET /api/v1/chutes/captive-portal/networks/wifi
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Type: application/json
{
"interface": "wlan0",
"type": "wifi",
"name": "wifi"
}
"""
cors.config_cors(request)
request.setHeader('Content-Type', 'application/json')
chute_obj = ChuteStorage.chuteList[chute]
networkInterfaces = chute_obj.getCache('networkInterfaces')
data = {}
for iface in networkInterfaces:
if iface['name'] != network:
continue
data = {
'name': iface['name'],
'type': iface['netType'],
'interface': iface['internalIntf']
}
return json.dumps(data)
@routes.route('/<chute>/networks/<network>/leases', methods=['GET'])
def get_leases(self, request, chute, network):
"""
Get current list of DHCP leases for chute network.
Returns a list of DHCP lease records with the following fields:
expires
lease expiration time (seconds since Unix epoch)
mac_addr
device MAC address
ip_addr
device IP address
hostname
name that the device reported
client_id
optional identifier supplied by device
**Example request**:
.. sourcecode:: http
GET /api/v1/chutes/captive-portal/networks/wifi/leases
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Type: application/json
[
{
"client_id": "01:5c:59:48:7d:b9:e6",
"expires": "1511816276",
"ip_addr": "192.168.128.64",
"mac_addr": "5c:59:48:7d:b9:e6",
"hostname": "paradrops-iPod"
}
]
"""
cors.config_cors(request)
request.setHeader('Content-Type', 'application/json')
chute_obj = ChuteStorage.chuteList[chute]
externalSystemDir = chute_obj.getCache('externalSystemDir')
leasefile = 'dnsmasq-{}.leases'.format(network)
path = os.path.join(externalSystemDir, leasefile)
# The format of the dnsmasq leases file is one entry per line with
# space-separated fields.
keys = ['expires', 'mac_addr', 'ip_addr', 'hostname', 'client_id']
leases = []
with open(path, 'r') as source:
for line in source:
parts = line.strip().split()
leases.append(dict(zip(keys, parts)))
return json.dumps(leases)
@routes.route('/<chute>/networks/<network>/ssid', methods=['GET'])
def get_ssid(self, request, chute, network):
"""
Get currently configured SSID for the chute network.
**Example request**:
.. sourcecode:: http
GET /api/v1/chutes/captive-portal/networks/wifi/ssid
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Type: application/json
{
"ssid": "Free WiFi",
"bssid": "02:00:08:24:03:dd"
}
"""
cors.config_cors(request)
request.setHeader('Content-Type', 'application/json')
chute_obj = ChuteStorage.chuteList[chute]
networkInterfaces = chute_obj.getCache('networkInterfaces')
ifname = None
for iface in networkInterfaces:
if iface['name'] == network:
ifname = iface['externalIntf']
break
address = os.path.join(settings.PDCONFD_WRITE_DIR, "hostapd", ifname)
return hostapd_control.execute(address, command="GET_CONFIG")
@routes.route('/<chute>/networks/<network>/ssid', methods=['PUT'])
def set_ssid(self, request, chute, network):
"""
Change the configured SSID for the chute network.
The change will not persist after a reboot. If a persistent change is
desired, you should update the chute configuration instead.
**Example request**:
.. sourcecode:: http
PUT /api/v1/chutes/captive-portal/networks/wifi/ssid
Content-Type: application/json
{
"ssid": "Best Free WiFi"
}
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Type: application/json
{
"message": "OK"
}
"""
cors.config_cors(request)
request.setHeader('Content-Type', 'application/json')
chute_obj = ChuteStorage.chuteList[chute]
networkInterfaces = chute_obj.getCache('networkInterfaces')
ifname = None
for iface in networkInterfaces:
if iface['name'] == network:
ifname = iface['externalIntf']
break
body = json.loads(request.content.read())
if "ssid" not in body:
raise Exception("ssid required")
command = "SET ssid {}".format(body['ssid'])
address = os.path.join(settings.PDCONFD_WRITE_DIR, "hostapd", ifname)
return hostapd_control.execute(address, command=command)
@routes.route('/<chute>/networks/<network>/hostapd_status',
methods=['GET'])
def get_hostapd_status(self, request, chute, network):
"""
Get low-level status information from the access point.
**Example request**:
.. sourcecode:: http
GET /api/v1/chutes/captive-portal/networks/wifi/hostapd_status
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Type: application/json
{
"olbc_ht": "0",
"cac_time_left_seconds": "N/A",
"num_sta_no_short_slot_time": "0",
"olbc": "1",
"num_sta_non_erp": "0",
"ht_op_mode": "0x4",
"state": "ENABLED",
"num_sta_ht40_intolerant": "0",
"channel": "11",
"bssid[0]": "02:00:08:24:03:dd",
"ieee80211n": "1",
"cac_time_seconds": "0",
"num_sta[0]": "1",
"ieee80211ac": "0",
"phy": "phy0",
"num_sta_ht_no_gf": "1",
"freq": "2462",
"num_sta_ht_20_mhz": "1",
"num_sta_no_short_preamble": "0",
"secondary_channel": "0",
"ssid[0]": "Free WiFi",
"num_sta_no_ht": "0",
"bss[0]": "vwlan7e1b"
}
"""
cors.config_cors(request)
request.setHeader('Content-Type', 'application/json')
chute_obj = ChuteStorage.chuteList[chute]
networkInterfaces = chute_obj.getCache('networkInterfaces')
ifname = None
for iface in networkInterfaces:
if iface['name'] == network:
ifname = iface['externalIntf']
break
address = os.path.join(settings.PDCONFD_WRITE_DIR, "hostapd", ifname)
return hostapd_control.execute(address, command="STATUS")
@routes.route('/<chute>/networks/<network>/stations', methods=['GET'])
def get_stations(self, request, chute, network):
"""
Get detailed information about connected wireless stations.
**Example request**:
.. sourcecode:: http
GET /api/v1/chutes/captive-portal/networks/wifi/stations
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Type: application/json
[
{
"rx_packets": "230",
"tdls_peer": "no",
"authenticated": "yes",
"rx_bytes": "12511",
"tx_bitrate": "1.0 MBit/s",
"tx_retries": "0",
"signal": "-45 [-49, -48] dBm",
"authorized": "yes",
"rx_bitrate": "65.0 MBit/s MCS 7",
"mfp": "no",
"tx_failed": "0",
"inactive_time": "4688 ms",
"mac_addr": "5c:59:48:7d:b9:e6",
"tx_bytes": "34176",
"wmm_wme": "yes",
"preamble": "short",
"tx_packets": "88",
"signal_avg": "-44 [-48, -47] dBm"
}
]
"""
cors.config_cors(request)
request.setHeader('Content-Type', 'application/json')
chute_obj = ChuteStorage.chuteList[chute]
networkInterfaces = chute_obj.getCache('networkInterfaces')
ifname = None
for iface in networkInterfaces:
if iface['name'] == network:
ifname = iface['externalIntf']
break
cmd = ['iw', 'dev', ifname, 'station', 'dump']
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
stations = []
current = {}
for line in proc.stdout:
line = line.strip()
match = re.match("Station\s+(\S+)\s+.*", line)
if match is not None:
current = {'mac_addr': match.group(1)}
stations.append(current)
continue
match = re.match("(.*):\s+(.*)", line)
if match is not None:
key = match.group(1).lower().replace(' ',
'_').replace('/', '_')
current[key] = match.group(2)
return json.dumps(stations)
@routes.route('/<chute>/networks/<network>/stations/<mac>',
methods=['GET'])
def get_station(self, request, chute, network, mac):
"""
Get detailed information about a connected station.
**Example request**:
.. sourcecode:: http
GET /api/v1/chutes/captive-portal/networks/wifi/stations/5c:59:48:7d:b9:e6
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Type: application/json
{
"rx_packets": "230",
"tdls_peer": "no",
"authenticated": "yes",
"rx_bytes": "12511",
"tx_bitrate": "1.0 MBit/s",
"tx_retries": "0",
"signal": "-45 [-49, -48] dBm",
"authorized": "yes",
"rx_bitrate": "65.0 MBit/s MCS 7",
"mfp": "no",
"tx_failed": "0",
"inactive_time": "4688 ms",
"mac_addr": "5c:59:48:7d:b9:e6",
"tx_bytes": "34176",
"wmm_wme": "yes",
"preamble": "short",
"tx_packets": "88",
"signal_avg": "-44 [-48, -47] dBm"
}
"""
cors.config_cors(request)
request.setHeader('Content-Type', 'application/json')
chute_obj = ChuteStorage.chuteList[chute]
networkInterfaces = chute_obj.getCache('networkInterfaces')
ifname = None
for iface in networkInterfaces:
if iface['name'] == network:
ifname = iface['externalIntf']
break
cmd = ['iw', 'dev', ifname, 'station', 'get', mac]
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
station = {}
for line in proc.stdout:
line = line.strip()
match = re.match("Station\s+(\S+)\s+.*", line)
if match is not None:
station['mac_addr'] = match.group(1)
continue
match = re.match("(.*):\s+(.*)", line)
if match is not None:
key = match.group(1).lower().replace(' ',
'_').replace('/', '_')
station[key] = match.group(2)
return json.dumps(station)
@routes.route('/<chute>/networks/<network>/stations/<mac>',
methods=['DELETE'])
def delete_station(self, request, chute, network, mac):
cors.config_cors(request)
request.setHeader('Content-Type', 'application/json')
chute_obj = ChuteStorage.chuteList[chute]
networkInterfaces = chute_obj.getCache('networkInterfaces')
ifname = None
for iface in networkInterfaces:
if iface['name'] == network:
ifname = iface['externalIntf']
break
cmd = ['iw', 'dev', ifname, 'station', 'del', mac]
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
messages = []
for line in proc.stdout:
line = line.strip()
messages.append(line)
return json.dumps(messages)
@routes.route('/<chute>/networks/<network>/hostapd_control/ws',
branch=True,
methods=['GET'])
def hostapd_control(self, request, chute, network):
chute_obj = ChuteStorage.chuteList[chute]
networkInterfaces = chute_obj.getCache('networkInterfaces')
ifname = None
for iface in networkInterfaces:
if iface['name'] == network:
ifname = iface['externalIntf']
break
ctrl_iface = os.path.join(settings.PDCONFD_WRITE_DIR, "hostapd",
ifname)
factory = hostapd_control.HostapdControlWSFactory(ctrl_iface)
factory.setProtocolOptions(autoPingInterval=10, autoPingTimeout=5)
return WebSocketResource(factory)
id="ltri-dota-gsi-money",
desc="Machine State for DOTA Money"),
"dota-gsi-money-inv":
Machine(name="dota-GSI",
iname="MONEY_INV",
id="ltri-dota-gsi-money-inv",
desc="Machine State for DOTA Money (inverted)"),
},
speed_enum={"FHUNDREDTHS": .05})
# lol
serialized = json.loads(
json.dumps(schema, indent=4, default=pydantic_encoder))
return serialized
webapp = Klein()
@webapp.route('/', methods=['POST'])
def index(request: Request):
content = json.loads(request.content.read())
try:
new_state = GameState(**content)
except Exception as e:
print(content)
print(e)
return "wew"
wamp_component.state = new_state
print(wamp_component.state.map.game_state)
class AuthApi(object):
routes = Klein()
def __init__(self, password_manager, token_manager):
self.password_manager = password_manager
self.token_manager = token_manager
@routes.route('/local', methods=['POST'])
def local_login(self, request):
"""
Login using local authentication (username+password).
"""
cors.config_cors(request)
request.setHeader('Content-Type', 'application/json')
body = json.loads(request.content.read())
username = body.get('username', self.password_manager.DEFAULT_USER_NAME)
password = body.get('password', self.password_manager.DEFAULT_PASSWORD)
success = self.password_manager.verify_password(username, password)
result = {
'username': username,
'success': success
}
if success:
token = self.token_manager.issue(username, domain="localhost",
role="admin")
result['token'] = token
else:
request.setResponseCode(401)
return json.dumps(result)
@routes.route('/cloud', methods=['POST'])
def auth_cloud(self, request):
"""
Login using credentials from the cloud controller.
This is an experimental new login method that lets users
present a token that they received from the cloud controller
as a login credential for a node. The idea is to enable easy
access for multiple developers to share a node, for example,
during a tutorial.
Instead of a username/password, the user presents a token received
from the cloud controller. The verify_cloud_token function
verifies the validity of the token with the controller, and if
successful, retrieves information about the bearer, particularly
the username and role. Finally, we generate a new token that
enables the user to authenticate with local API endpoints.
"""
cors.config_cors(request)
request.setHeader('Content-Type', 'application/json')
body = json.loads(request.content.read())
token = body.get('token', '')
headers = {
"Authorization": "Bearer {}".format(token)
}
node_id = nexus.core.info.pdid
# Pass custom Authorization header and setAuthHeader=False to prevent
# PDServerRequest from using the node's own authorization token.
d1 = PDServerRequest("/api/users/me",
headers=headers,
setAuthHeader=False
).get()
d2 = PDServerRequest("/api/routers/{}".format(node_id),
headers=headers,
setAuthHeader=False
).get()
def token_verified(responses):
for response in responses:
if not response.success or response.data is None:
request.setResponseCode(401)
return
remote_user = responses[0].data
node_info = responses[1].data
role = get_access_level(remote_user, node_info)
token = self.token_manager.issue(remote_user['email'],
domain="paradrop.org", role=role)
result = {
'token': token,
'username': remote_user['email']
}
return json.dumps(result)
d = defer.gatherResults([d1, d2])
d.addCallback(token_verified)
return d
class ProxyApp(object):
app = Klein()
ns = "{http://www.yale.edu/tp/cas}"
port = None
logout_instant_skew = 5
ticket_name = 'ticket'
service_name = 'service'
renew_name = 'renew'
pgturl_name = 'pgtUrl'
reactor = reactor
auth_info_resource = None
auth_info_callback = None
remoteUserHeader = 'Remote-User'
logout_patterns = None
logout_passthrough = False
verbose = False
proxy_client_endpoint_s = None
cas_client_endpoint_s = None
def __init__(self,
proxied_url,
cas_info,
fqdn=None,
authorities=None,
plugins=None,
is_https=True,
excluded_resources=None,
excluded_branches=None,
remote_user_header=None,
logout_patterns=None,
logout_passthrough=False,
template_dir=None,
template_resource='/_templates',
proxy_client_endpoint_s=None,
cas_client_endpoint_s=None):
self.proxy_client_endpoint_s = proxy_client_endpoint_s
self.cas_client_endpoint_s = cas_client_endpoint_s
self.logout_passthrough = logout_passthrough
self.template_dir = template_dir
if template_dir is not None:
self.template_loader_ = FileSystemLoader(template_dir)
self.template_env_ = Environment()
self.templateStaticResource_ = self.create_template_static_resource(
)
if template_resource is not None:
if not template_resource.endswith('/'):
template_resource = "{0}/".format(template_resource)
if template_resource is not None and template_dir is not None:
static_base = "{0}static/".format(template_resource)
self.static = self.app.route(static_base,
branch=True)(self.__class__.static)
self.static_base = static_base
self.template_resource = template_resource
if logout_patterns is not None:
self.logout_patterns = [
parse_url_pattern(pattern) for pattern in logout_patterns
]
for pattern in self.logout_patterns:
assert pattern is None or pattern.scheme == '', (
"Logout pattern '{0}' must be a relative URL.".format(pattern))
if remote_user_header is not None:
self.remoteUserHeader = remote_user_header
self.excluded_resources = excluded_resources
self.excluded_branches = excluded_branches
self.is_https = is_https
if proxied_url.endswith('/'):
proxied_url = proxied_url[:-1]
self.proxied_url = proxied_url
p = urlparse.urlparse(proxied_url)
self.p = p
self.proxied_scheme = p.scheme
netloc = p.netloc
self.proxied_netloc = netloc
self.proxied_host = netloc.split(':')[0]
self.proxied_path = p.path
self.cas_info = cas_info
cas_param_names = set([])
cas_param_names.add(self.ticket_name.lower())
cas_param_names.add(self.service_name.lower())
cas_param_names.add(self.renew_name.lower())
cas_param_names.add(self.pgturl_name.lower())
self.cas_param_names = cas_param_names
if fqdn is None:
fqdn = socket.getfqdn()
self.fqdn = fqdn
self.valid_sessions = {}
self.logout_tickets = {}
self._make_agents(authorities)
# Sort/tag plugins
if plugins is None:
plugins = []
content_modifiers = []
info_acceptors = []
cas_redirect_handlers = []
interceptors = []
access_control = []
for plugin in plugins:
if IResponseContentModifier.providedBy(plugin):
content_modifiers.append(plugin)
if IRProxyInfoAcceptor.providedBy(plugin):
info_acceptors.append(plugin)
if ICASRedirectHandler.providedBy(plugin):
cas_redirect_handlers.append(plugin)
if IResourceInterceptor.providedBy(plugin):
interceptors.append(plugin)
if IAccessControl.providedBy(plugin):
access_control.append(plugin)
self.info_acceptors = info_acceptors
content_modifiers.sort(key=lambda x: x.mod_sequence)
self.content_modifiers = content_modifiers
cas_redirect_handlers.sort(key=lambda x: x.cas_redirect_sequence)
self.cas_redirect_handlers = cas_redirect_handlers
interceptors.sort(key=lambda x: x.interceptor_sequence)
self.interceptors = interceptors
access_control.sort(key=lambda x: x.ac_sequence)
self.access_control = access_control
# Create static resources.
static_resources = {}
for plugin in plugins:
if IStaticResourceProvider.providedBy(plugin):
if plugin.static_resource_base in static_resources:
if static_resources[
plugin.
static_resource_base] != plugin.static_resource_dir:
raise Exception(
"Static resource conflict for '{0}': '{1}' != '{2}'"
.format(
plugin.static_resource_base,
static_resources[plugin.static_resource_base],
plugin.static_resource_dir))
else:
static_resources[
plugin.
static_resource_base] = plugin.static_resource_dir
self.static_handlers = []
for n, (resource_base,
resource_dir) in enumerate(static_resources.iteritems()):
handler = lambda self, request: File(resource_dir)
handler = self.app.route(resource_base, branch=True)(handler)
self.static_handlers.append(handler)
def log(self, msg, important=False):
if important or self.verbose:
if important:
tag = "INFO"
else:
tag = "DEBUG"
log.msg("[{0}] {1}".format(tag, msg))
def handle_port_set(self):
fqdn = self.fqdn
port = self.port
proxied_scheme = self.proxied_scheme
proxied_netloc = self.proxied_netloc
proxied_path = self.proxied_path
for plugin in self.info_acceptors:
plugin.proxy_fqdn = fqdn
plugin.proxy_port = port
plugin.proxied_scheme = proxied_scheme
plugin.proxied_netloc = proxied_netloc
plugin.proxied_path = proxied_path
plugin.handle_rproxy_info_set()
plugin.expire_session = self._expired
def _make_agents(self, auth_files):
"""
Configure the web clients that:
* perform backchannel CAS ticket validation
* proxy the target site
"""
self.connectionPool = HTTPConnectionPool(self.reactor)
if auth_files is None or len(auth_files) == 0:
agent = Agent(self.reactor, pool=self.connectionPool)
else:
extra_ca_certs = []
for ca_cert in auth_files:
with open(ca_cert, "rb") as f:
data = f.read()
cert = crypto.load_certificate(crypto.FILETYPE_PEM, data)
del data
extra_ca_certs.append(cert)
policy = CustomPolicyForHTTPS(extra_ca_certs)
agent = Agent(self.reactor,
contextFactory=policy,
pool=self.connectionPool)
if self.proxy_client_endpoint_s is not None:
self.proxyConnectionPool = HTTPConnectionPool(self.reactor)
self.proxy_agent = Agent.usingEndpointFactory(
self.reactor,
WebClientEndpointFactory(self.reactor,
self.proxy_client_endpoint_s),
pool=self.proxyConnectionPool)
else:
self.proxy_agent = agent
if self.cas_client_endpoint_s is not None:
self.casConnectionPool = HTTPConnectionPool(self.reactor)
self.cas_agent = Agent.usingEndpointFactory(
self.reactor,
WebClientEndpointFactory(self.reactor,
self.cas_client_endpoint_s),
pool=self.casConnectionPool)
else:
self.cas_agent = agent
def is_excluded(self, request):
resource = request.path
if resource in self.excluded_resources:
return True
for excluded in self.excluded_branches:
if proxyutils.is_resource_or_child(excluded, resource):
return True
return False
def mod_headers(self, h):
keymap = {}
for k, v in h.iteritems():
key = k.lower()
if key in keymap:
keymap[key].append(k)
else:
keymap[key] = [k]
if 'host' in keymap:
for k in keymap['host']:
h[k] = [self.proxied_netloc]
if 'origin' in keymap:
for k in keymap['origin']:
h[k] = [self.proxied_netloc]
if 'content-length' in keymap:
for k in keymap['content-length']:
del h[k]
if 'referer' in keymap:
referer_handled = False
keys = keymap['referer']
if len(keys) == 1:
k = keys[0]
values = h[k]
if len(values) == 1:
referer = values[0]
new_referer = self.proxy_url_to_proxied_url(referer)
if new_referer is not None:
h[k] = [new_referer]
self.log("Re-wrote Referer header: '%s' => '%s'" %
(referer, new_referer))
referer_handled = True
if not referer_handled:
for k in keymap['referer']:
del h[k]
return h
def _check_for_logout(self, request):
data = request.content.read()
samlp_ns = "{urn:oasis:names:tc:SAML:2.0:protocol}"
try:
root = etree.fromstring(data)
except Exception as ex:
root = None
if (root is not None) and (root.tag == "%sLogoutRequest" % samlp_ns):
instant = root.get('IssueInstant')
if instant is not None:
try:
instant = parse_date(instant)
except ValueError:
self.log("Invalid issue_instant supplied: '{0}'.".format(
instant),
important=True)
instant = None
if instant is not None:
utcnow = datetime.datetime.utcnow()
seconds = abs(
(utcnow -
instant.replace(tzinfo=None)).total_seconds())
if seconds <= self.logout_instant_skew:
results = root.findall("%sSessionIndex" % samlp_ns)
if len(results) == 1:
result = results[0]
ticket = result.text
sess_uid = self.logout_tickets.get(ticket, None)
if sess_uid is not None:
self._expired(sess_uid)
return True
else:
self.log(
("No matching session for logout request "
"for ticket '{0}'.").format(ticket))
else:
self.log(("Issue instant was not within"
" {0} seconds of actual time.").format(
self.logout_instant_skew),
important=True)
else:
self.log("Could not parse issue instant.", important=True)
else:
self.log("'IssueInstant' attribute missing from root.",
important=True)
elif root is None:
self.log("Could not parse XML.", important=True)
return False
@app.route("/", branch=True)
def proxy(self, request):
for pattern in self.logout_patterns:
if does_url_match_pattern(request.uri, pattern):
sess = request.getSession()
sess_uid = sess.uid
self._expired(sess_uid)
cas_logout = self.cas_info.get('logout_url', None)
if cas_logout is not None:
if self.logout_passthrough:
d = self.reverse_proxy(request, protected=False)
return request.redirect(cas_logout)
else:
return self.reverse_proxy(request, protected=False)
if self.is_excluded(request):
return self.reverse_proxy(request, protected=False)
valid_sessions = self.valid_sessions
sess = request.getSession()
sess_uid = sess.uid
if not sess_uid in valid_sessions:
self.log(("Session {0} not in valid sessions. "
"Will authenticate with CAS.").format(sess_uid))
if request.method == 'POST':
headers = request.requestHeaders
if headers.hasHeader("Content-Type"):
ct_list = headers.getRawHeaders("Content-Type")
#log.msg("[DEBUG] ct_list: %s" % str(ct_list))
for ct in ct_list:
if ct.find('text/xml') != -1 or ct.find(
'application/xml') != -1:
if self._check_for_logout(request):
return ""
else:
break
# CAS Authentication
# Does this request have a ticket? I.e. is it coming back from a successful
# CAS authentication?
args = request.args
ticket_name = self.ticket_name
if ticket_name in args:
values = args[ticket_name]
if len(values) == 1:
ticket = values[0]
d = self.validate_ticket(ticket, request)
return d
# If no ticket is present, redirect to CAS.
d = self.redirect_to_cas_login(request)
return d
elif request.path == self.auth_info_resource:
self.log("Providing authentication info.")
return self.deliver_auth_info(request)
else:
d = self.reverse_proxy(request)
return d
def deliver_auth_info(self, request):
valid_sessions = self.valid_sessions
sess = request.getSession()
sess_uid = sess.uid
session_info = valid_sessions[sess_uid]
username = session_info['username']
attributes = session_info['attributes']
doc = {'username': username, 'attributes': attributes}
serialized = json.dumps(doc)
request.responseHeaders.setRawHeaders('Content-Type',
['application/json'])
return serialized
def get_url(self, request):
if self.is_https:
scheme = 'https'
default_port = 443
else:
scheme = 'http'
default_port = 80
fqdn = self.fqdn
port = self.port
if port is None:
port = default_port
if port == default_port:
return urlparse.urljoin("%s://%s" % (scheme, fqdn), request.uri)
else:
return urlparse.urljoin("%s://%s:%d" % (scheme, fqdn, port),
request.uri)
def redirect_to_cas_login(self, request):
"""
Begin the CAS redirection process.
"""
service_url = self.get_url(request)
d = None
for plugin in self.cas_redirect_handlers:
if d is None:
d = defer.maybeDeferred(plugin.intercept_service_url,
service_url, request)
else:
d.addCallback(plugin.intercept_service_url, request)
if d is None:
return self.complete_redirect_to_cas_login(service_url, request)
else:
d.addCallback(self.complete_redirect_to_cas_login, request)
return d
def complete_redirect_to_cas_login(self, service_url, request):
"""
Complete the CAS redirection process.
Return a deferred that will redirect the user-agent to the CAS login.
"""
cas_info = self.cas_info
login_url = cas_info['login_url']
p = urlparse.urlparse(login_url)
params = {self.service_name: service_url}
if p.query == '':
param_str = urlencode(params)
else:
qs_map = urlparse.parse_qs(p.query)
qs_map.update(params)
param_str = urlencode(qs_map, doseq=True)
p = urlparse.ParseResult(*tuple(p[:4] + (param_str, ) + p[5:]))
url = urlparse.urlunparse(p)
self.log("Redirecting to CAS with URL '{0}'.".format(url))
d = request.redirect(url)
return d
def validate_ticket(self, ticket, request):
service_name = self.service_name
ticket_name = self.ticket_name
this_url = self.get_url(request)
p = urlparse.urlparse(this_url)
qs_map = urlparse.parse_qs(p.query)
if ticket_name in qs_map:
del qs_map[ticket_name]
param_str = urlencode(qs_map, doseq=True)
p = urlparse.ParseResult(*tuple(p[:4] + (param_str, ) + p[5:]))
service_url = urlparse.urlunparse(p)
params = {
service_name: service_url,
ticket_name: ticket,
}
param_str = urlencode(params, doseq=True)
p = urlparse.urlparse(self.cas_info['service_validate_url'])
p = urlparse.ParseResult(*tuple(p[:4] + (param_str, ) + p[5:]))
service_validate_url = urlparse.urlunparse(p)
self.log("Requesting service-validate URL => '{0}' ...".format(
service_validate_url))
http_client = HTTPClient(self.cas_agent)
d = http_client.get(service_validate_url)
d.addCallback(treq.content)
d.addCallback(self.parse_sv_results, service_url, ticket, request)
return d
def parse_sv_results(self, payload, service_url, ticket, request):
self.log("Parsing /serviceValidate results ...")
ns = self.ns
try:
root = etree.fromstring(payload)
except (etree.XMLSyntaxError, ) as ex:
self.log(("error='Error parsing XML payload.' "
"service='{0}' ticket={1}'/n{2}").format(
service_url, ticket, ex),
important=True)
return self.render_template_500(request)
if root.tag != ('%sserviceResponse' % ns):
self.log(
("error='Error parsing XML payload. No `serviceResponse`.' "
"service='{0}' ticket={1}'").format(service_url, ticket),
important=True)
return self.render_template_403(request)
results = root.findall("{0}authenticationSuccess".format(ns))
if len(results) != 1:
self.log((
"error='Error parsing XML payload. No `authenticationSuccess`.' "
"service='{0}' ticket={1}'").format(service_url, ticket),
important=True)
return self.render_template_403(request)
success = results[0]
results = success.findall("{0}user".format(ns))
if len(results) != 1:
self.log(
("error='Error parsing XML payload. Not exactly 1 `user`.' "
"service='{0}' ticket={1}'").format(service_url, ticket),
important=True)
return self.render_template_403(request)
user = results[0]
username = user.text
attributes = success.findall("{0}attributes".format(ns))
attrib_map = {}
for attrib_container in attributes:
for elm in attrib_container.findall('./*'):
tag_name = elm.tag[len(ns):]
value = elm.text
attrib_map.setdefault(tag_name, []).append(value)
# Access control plugins
access_control = self.access_control
for ac_plugin in access_control:
is_allowed, reason = ac_plugin.isAllowed(username, attrib_map)
if not is_allowed:
self.log((
"Access denied: user='******' ac_plugin='{ac_plugin}' "
"reason={reason}, service='{service}' ticket='{ticket}'"
).format(username=username,
ac_plugin=ac_plugin.tagname,
service=service_url,
ticket=ticket,
reason=reason),
important=True)
return self.render_template_403(request,
username=username,
reason=reason)
# Update session session
valid_sessions = self.valid_sessions
logout_tickets = self.logout_tickets
sess = request.getSession()
sess_uid = sess.uid
if sess_uid not in valid_sessions:
valid_sessions[sess_uid] = {}
valid_sessions[sess_uid].update({
'username': username,
'ticket': ticket,
'attributes': attrib_map
})
if not ticket in logout_tickets:
logout_tickets[ticket] = sess_uid
auth_info_callback = self.auth_info_callback
if auth_info_callback is not None:
auth_info_callback(username, attrib_map)
sess.notifyOnExpire(lambda: self._expired(sess_uid))
# Reverse proxy.
return request.redirect(service_url)
def _expired(self, uid):
valid_sessions = self.valid_sessions
if uid in valid_sessions:
session_info = valid_sessions[uid]
username = session_info['username']
ticket = session_info['ticket']
del valid_sessions[uid]
auth_info_callback = self.auth_info_callback
if auth_info_callback is not None:
auth_info_callback(username, None)
logout_tickets = self.logout_tickets
if ticket in logout_tickets:
del logout_tickets[ticket]
self.log(("label='Expired session.' session_id='{0}' "
"username='******'").format(uid, username))
def reverse_proxy(self, request, protected=True):
if protected:
sess = request.getSession()
valid_sessions = self.valid_sessions
sess_uid = sess.uid
username = valid_sessions[sess_uid]['username']
# Normal reverse proxying.
kwds = {}
cookiejar = {}
kwds['allow_redirects'] = False
kwds['cookies'] = cookiejar
req_headers = self.mod_headers(
dict(request.requestHeaders.getAllRawHeaders()))
kwds['headers'] = req_headers
if protected:
kwds['headers'][self.remoteUserHeader] = [username]
if request.method in ('PUT', 'POST'):
kwds['data'] = request.content.read()
url = self.proxied_url + request.uri
# Determine if a plugin wants to intercept this URL.
interceptors = self.interceptors
for interceptor in interceptors:
if interceptor.should_resource_be_intercepted(
url, request.method, req_headers, request):
return interceptor.handle_resource(url, request.method,
req_headers, request)
# Check if this is a request for a websocket.
d = self.checkForWebsocketUpgrade(request)
if d is not None:
return d
# Typical reverse proxying.
self.log("Proxying URL => {0}".format(url))
http_client = HTTPClient(self.proxy_agent)
d = http_client.request(request.method, url, **kwds)
def process_response(response, request):
req_resp_headers = request.responseHeaders
resp_code = response.code
resp_headers = response.headers
resp_header_map = dict(resp_headers.getAllRawHeaders())
# Rewrite Location headers for redirects as required.
if resp_code in (301, 302, 303, 307,
308) and "Location" in resp_header_map:
values = resp_header_map["Location"]
if len(values) == 1:
location = values[0]
if request.isSecure():
proxy_scheme = 'https'
else:
proxy_scheme = 'http'
new_location = self.proxied_url_to_proxy_url(
proxy_scheme, location)
if new_location is not None:
resp_header_map['Location'] = [new_location]
request.setResponseCode(response.code, message=response.phrase)
for k, v in resp_header_map.iteritems():
if k == 'Set-Cookie':
v = self.mod_cookies(v)
req_resp_headers.setRawHeaders(k, v)
return response
def mod_content(body, request):
"""
Modify response content before returning it to the user agent.
"""
d = None
for content_modifier in self.content_modifiers:
if d is None:
d = content_modifier.transform_content(body, request)
else:
d.addCallback(content_modifier.transform_content, request)
if d is None:
return body
else:
return d
d.addCallback(process_response, request)
d.addCallback(treq.content)
d.addCallback(mod_content, request)
return d
def checkForWebsocketUpgrade(self, request):
def _extract(name):
raw_value = request.getHeader(name)
if raw_value is None:
return set([])
else:
return set(raw_value.split(', '))
upgrade = _extract("Upgrade")
connection = _extract("Connection")
if 'websocket' in upgrade and 'Upgrade' in connection:
uri = request.uri
proxy_fqdn = self.fqdn
proxy_port = self.port
proxied_scheme = self.proxied_scheme
proxied_netloc = self.proxied_netloc
proxied_path = self.proxied_path
if self.is_https:
scheme = 'wss'
else:
scheme = 'ws'
netloc = "{0}:{1}".format(proxy_fqdn, proxy_port)
proxy_url = "{0}://{1}{2}".format(scheme, netloc, request.uri)
if proxied_scheme == 'https':
proxied_scheme = 'wss'
else:
proxied_scheme = 'ws'
proxied_url = proxyutils.proxy_url_to_proxied_url(
proxied_scheme,
proxy_fqdn,
proxy_port,
proxied_netloc,
proxied_path,
proxy_url,
)
origin = proxied_url
kind = "tcp"
if proxied_scheme == 'wss':
kind = 'ssl'
parts = proxied_netloc.split(":", 1)
proxied_host = parts[0]
if len(parts) == 2:
proxied_port = int(parts[1])
elif proxied_scheme == 'wss':
proxied_port = 443
else:
proxied_port = 80
extra = "" #TODO: SSL options.
proxied_endpoint_str = "{0}:host={1}:port={2}{3}".format(
kind, proxied_host, proxied_port, extra)
if proxied_url is not None:
resource = makeWebsocketProxyResource(proxy_url,
proxied_endpoint_str,
proxied_url,
request,
reactor=self.reactor,
origin=origin,
verbose=self.verbose)
return resource
return None
def mod_cookies(self, value_list):
proxied_path = self.proxied_path
proxied_path_size = len(proxied_path)
results = []
for cookie_value in value_list:
c = Cookie.SimpleCookie()
c.load(cookie_value)
for k in c.keys():
m = c[k]
if m.has_key('path'):
m_path = m['path']
if self.is_proxy_path_or_child(m_path):
m_path = m_path[proxied_path_size:]
m['path'] = m_path
results.append(c.output(header='')[1:])
return results
def is_proxy_path_or_child(self, path):
return proxyutils.is_proxy_path_or_child(self.proxied_path, path)
def proxied_url_to_proxy_url(self, proxy_scheme, target_url):
return proxyutils.proxied_url_to_proxy_url(proxy_scheme, self.fqdn,
self.port,
self.proxied_netloc,
self.proxied_path,
target_url)
def proxy_url_to_proxied_url(self, target_url):
return proxyutils.proxy_url_to_proxied_url(self.proxied_scheme,
self.fqdn, self.port,
self.proxied_netloc,
self.proxied_path,
target_url)
def get_template_static_base(self):
if self.template_resource is None:
return None
else:
return '{0}static/'.format(self.template_resource)
def render_template_403(self, request, **kwargs):
template_dir = self.template_dir
if template_dir is None:
request.setResponseCode(403)
return ""
else:
return self.render_template('error/403.jinja2',
request=request,
**kwargs)
def render_template_500(self, request, **kwargs):
template_dir = self.template_dir
if template_dir is None:
request.setResponseCode(500)
return ""
else:
return self.render_template('error/500.jinja2',
request=request,
**kwargs)
def render_template(self, template_name, **kwargs):
template_dir = self.template_dir
try:
template = self.template_loader_.load(self.template_env_,
template_name)
except TemplateNotFound:
raise Exception(
"The template '{0}' was not found.".format(template_name))
return template.render(static_base=self.static_base,
**kwargs).encode('utf-8')
def create_template_static_resource(self):
static_path = os.path.join(self.template_dir, 'static')
static = File(static_path)
return static
def static(self, request):
return self.templateStaticResource_
@app.handle_errors(Exception)
def handle_uncaught_errors(self, request, failure):
self.log("Uncaught exception: {0}".format(failure), important=True)
return self.render_template_500(request=request, failure=failure)
# #!/usr/bin/python3
from klein import run, route, Klein
import json
import os
import pickle
app = Klein()
# nlp = spacy.load('en_core_web_sm')
# def tokeni(sentence):
# return list([str(x) for x in nlp.tokenizer(sentence)])
def load_model(filename):
with open(filename, 'rb') as f:
model_struct = pickle.load(f)
clf = model_struct['model']
return clf
def predict_class_from_text(clf,text):
predicted = clf.predict([text])[0]
return predicted
def predict_proba_from_text(clf,text):
proba = clf.predict_proba([text])[0][1]
return int(proba*100)
@app.route("/ecommerce-category", methods=['POST'])
def get_category(request):
content = json.loads(request.content.read())
text = content['content']
from klein import Klein
from twisted.internet import task, reactor
from twisted.web.resource import Resource
import Sequential
app = Klein()
def delayedCall(n):
return 'Delayed for %d seconds' % n
@app.route('/delay/<int:t>')
def delay(requests, t):
"""
Delay the response by t seconds
"""
if t > 0:
d = task.deferLater(reactor, t, delayedCall, t)
return d
else:
return delayedCall(0)
@app.route('/upload', branch=True)
def sequential(request):
return Sequential.app.resource()
app.run('localhost', 8000)
class Api: # noqa: D101
app = Klein()
default = { # note that these defaults are documented in configuration/www.rst
"left_pad": 5,
"left_text": "Build Status",
"left_color": "#555",
"right_pad": 5,
"border_radius": 5,
"style": "plastic",
"template_name": "{style}.svg.j2",
"font_face": "DejaVu Sans",
"font_size": 11,
"color_scheme": color_scheme,
}
def __init__(self, ep):
self.ep = ep
self.env = jinja2.Environment(
loader=jinja2.
ChoiceLoader([jinja2.PackageLoader("buildbot_badges"), jinja2.FileSystemLoader("templates")])
)
def makeConfiguration(self, request): # noqa: D102
config = {}
config.update(self.default)
for k, v in self.ep.config.items():
if k == "color_scheme":
config[k].update(v) # type: ignore
else:
config[k] = v
for k, v in request.args.items():
k = bytes2unicode(k)
config[k] = escape(bytes2unicode(v[0]))
return config
@app.route("/<path:builder>.png", methods=["GET"])
@defer.inlineCallbacks
def getPng(self, request, builder): # noqa: D102
svg = yield self.getSvg(request, builder)
request.setHeader("content-type", "image/png")
defer.returnValue(cairosvg.svg2png(svg))
@app.route("/<path:builder>.svg", methods=["GET"])
@defer.inlineCallbacks
def getSvg(self, request, builder): # noqa: D102
master: buildbot.master.BuildMaster = self.ep.master
if not builder.isidentifier():
builder = yield master.db.builders.findBuilderId(builder, False)
config = self.makeConfiguration(request)
request.setHeader("content-type", "image/svg+xml")
request.setHeader("cache-control", "no-cache")
# get the last build for that builder using the data api
last_build = yield self.ep.master.data.get(("builders", builder, "builds"), limit=1, order=["-number"])
# get the status text corresponding to results code
results_txt = "unknown"
if last_build:
results = last_build[0]["results"]
complete = last_build[0]["complete"]
if not complete:
results_txt = "running"
elif results >= 0 and results < len(Results):
results_txt = Results[results]
svgdata = self.makesvg(results_txt, results_txt, left_text=config["left_text"], config=config)
defer.returnValue(svgdata)
def textwidth(self, text, config):
"""
Calculates the width of the specified text.
"""
surface = cairo.SVGSurface(None, 1280, 200)
ctx = cairo.Context(surface)
ctx.select_font_face(config["font_face"], cairo.FONT_SLANT_NORMAL, cairo.FONT_WEIGHT_NORMAL)
ctx.set_font_size(int(config["font_size"]))
return ctx.text_extents(text)[4]
def makesvg(self, right_text, status=None, left_text=None, left_color=None, config=None):
"""
Renders an SVG from the template, using the specified data.
"""
right_color = config["color_scheme"].get(status, "#9f9f9f") # Grey
left_text = left_text or config["left_text"]
left_color = left_color or config["left_color"]
left = {"color": left_color, "text": left_text, "width": self.textwidth(left_text, config)}
right = {"color": right_color, "text": right_text, "width": self.textwidth(right_text, config)}
template = self.env.get_template(config["template_name"].format(**config))
return template.render(left=left, right=right, config=config)
class orderResource(object):
app = Klein()
def __init__(self, dbPools, mqPools):
self.dbPools = dbPools
self.mqPools = mqPools
@app.route('/<path:catchall>')
def catchAll(self, request, catchall):
request.setHeader('Content-Type', 'application/json')
request.setResponseCode(404)
return responseJson(
API_RESPONSE_CODE_ENDPOINT_NOT_EXISTS,
API_RESPONSE_MSG.get(API_RESPONSE_CODE_ENDPOINT_NOT_EXISTS))
@app.route('/')
def index(self, request):
request.setHeader('Content-Type', 'application/json')
request.setResponseCode(404)
return responseJson(API_RESPONSE_CODE_SUCCESS,
API_RESPONSE_MSG.get(API_RESPONSE_CODE_SUCCESS))
@app.route('/order/create/<string:name>', methods=['POST'])
def create(self, request, name):
def createOrderSuccess(result, tradeNo):
if False != result:
log.msg('[apiService, %s] %s: order create success' % (
name,
tradeNo,
))
publishData = {'agentUid': result[0], 'tradeNo': result[1]}
self.publishOrderData(name, tradeNo, dictToJson(publishData))
else:
log.msg('[apiService, %s] %s: order create success' % (
name,
tradeNo,
))
log.msg('[apiService, %s] %s: result:' % (
name,
tradeNo,
), result)
def createOrderFail(error, tradeNo):
log.msg('[apiService, %s] %s: order create fail' % (
name,
tradeNo,
))
log.msg('[apiService, %s] %s: error:' % (
name,
tradeNo,
), error)
def createOrder(txn, productId, agentUid, tradeNo):
ret = False
tradeGoodsTbName = getAppConfig('%s.table.trade_goods' % (name, ))
tradeOrderTbName = getAppConfig('%s.table.trade_order' % (name, ))
agentUserTbName = getAppConfig('%s.table.agent_user' % (name, ))
qUserTbName = getAppConfig('%s.table.q_user' % (name, ))
txn.execute(
'SELECT name,price,amount,bonus,gameid FROM ' +
tradeGoodsTbName +
' WHERE autoid = %s AND status = 1 AND level = 2',
(productId, ))
goodsData = txn.fetchone()
if goodsData is not None:
txn.execute(
'SELECT u.uid, u.username, m.uid AS pagentid, m.code AS pcode FROM '
+ agentUserTbName + ' AS m INNER JOIN ' + qUserTbName +
' AS u ON u.agentid = m.uid WHERE m.uid = %s',
(agentUid, ))
userData = txn.fetchone()
if userData is not None:
txn.execute(
'INSERT INTO ' + tradeOrderTbName +
'(`uid`, `appuname`, `goodid`, `tradeno`, `tradetype`, `coin`, `pagentid`, `pcode`, `amount`) VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s)',
(userData.get('uid'), userData.get('username'),
productId, tradeNo,
'AGENTPAY', goodsData.get('price'),
userData.get('pagentid'), userData.get('pcode'),
goodsData.get('amount') + goodsData.get('bonus')))
ret = (agentUid, tradeNo)
return ret
tradeNo = "%s%s%s" % (
10,
getGameIdBySymbol(name),
getTradeNo(),
)
log.msg('[apiService, %s] %s: order create parms' % (
name,
tradeNo,
), request.args)
parmsRule = {
'agent_uid': {
'name': requestValidation.RULE_NAME_REQUIRED,
'error': API_RESPONSE_CODE_INVALID_AGENT_UID
},
'good_id': {
'name': requestValidation.RULE_NAME_REQUIRED,
'error': API_RESPONSE_CODE_INVALID_PRODUCTID
},
'timestamp': {
'name': requestValidation.RULE_NAME_NUMBER,
'error': API_RESPONSE_CODE_INVALID_TIMESTAMP
},
'sign': {
'name': requestValidation.RULE_NAME_REQUIRED,
'error': API_RESPONSE_CODE_INVALID_SIGN
}
}
gameId = getGameIdBySymbol(name)
if gameId is None:
ret = API_RESPONSE_CODE_INVALID_GAMEID
else:
_requestValidation = requestValidation(request, parmsRule)
if True == _requestValidation.validation():
parmsSign = request.args.get('sign')[0]
parms = request.args
parms.pop('sign')
if True != identifySign(parms, parmsSign):
ret = API_RESPONSE_CODE_INVALID_SIGN
else:
productId = request.args.get('good_id')[0]
agentUid = request.args.get('agent_uid')[0]
log.msg(self.dbPools)
dbDefer = self.dbPools[name].runInteraction(
createOrder, productId, agentUid, tradeNo)
dbDefer.addCallback(createOrderSuccess, tradeNo)
dbDefer.addErrback(createOrderFail, tradeNo)
ret = API_RESPONSE_CODE_SUCCESS
else:
ret = _requestValidation.getErrorCode()
request.setHeader('Content-Type', 'application/json')
return responseJson(ret, API_RESPONSE_MSG.get(ret))
@app.route('/order/info/<string:name>')
def info(self, request, name):
request.setHeader('Content-Type', 'application/json')
return responseJson(API_RESPONSE_CODE_SUCCESS,
API_RESPONSE_MSG.get(API_RESPONSE_CODE_SUCCESS))
def publishOrderData(self, game, tradeNo, data):
try:
mqConn = self.mqPools[game]
mqChannel = mqConn.channel()
queueConfig = getMqQueueConfig(game)
exchange = queueConfig.get('exchange')
queueName = queueConfig.get('queueName')
routingKey = queueConfig.get('routingKey')
mqChannel.confirm_delivery()
mqChannel.exchange_declare(exchange=exchange, durable=True)
mqChannel.queue_declare(queue=queueName, durable=True)
mqChannel.queue_bind(queue=queueName,
exchange=exchange,
routing_key=routingKey)
mqChannel.publish(exchange=exchange,
routing_key=routingKey,
body=data)
log.msg(
'[apiService, %s] %s: order publish success:' % (
game,
tradeNo,
), data)
except Exception, e:
log.msg(
'[apiService, %s] %s: order publish fail: %s' % (
game,
tradeNo,
), data, e.message)
log.msg(e)
finally:
def setUp(self):
self.app = Klein()
self.kr = KleinResource(self.app)
class _One(object):
oneKlein = Klein()
@oneKlein.route("/foo")
def foo(self):
pass
# This is our WAMP application
##
wampapp = Application(u'com.example')
@wampapp.register()
def square(x):
print("square() called with {}".format(x))
return x * x
# This is our Web application
##
webapp = Klein()
webapp.visits = 0
webapp.templates = jinja2.Environment(loader=jinja2.FileSystemLoader('templates'))
@webapp.route('/')
def home(request):
webapp.visits += 1
wampapp.session.publish(u'com.example.onvisit', visits=webapp.visits)
page = webapp.templates.get_template('index.html')
return page.render(visits=webapp.visits)
@webapp.route('/square/<int:x>')
@inlineCallbacks
def square(request, x):
import treq
from klein import Klein
from twisted.internet.defer import inlineCallbacks, returnValue
from bs4 import BeautifulSoup
import json
import re
from twisted.python import log
app = Klein()
comp_name_regex = re.compile(r'\b[\w ]+\b')
def clean_text(str,flag='td'):
if flag == 'td':
return str.find('td').get_text().replace('-',' ').replace(',','').replace('.','').strip().encode('utf-8','ignore')
elif flag == 'th':
return str.find('th').get_text().strip().replace('-',' ').replace(',','').replace('.','').split(':')[0].encode('utf-8','ignore')
@app.route('/')
def helloworld():
return "<h>Helloe!!<\h>"
@app.route('/apis/get',methods=['GET',])
@inlineCallbacks
def check_status(request):
if not request.args.get("cname"):
pass
# abort(400)
# elif not re.match(request.args.get("cname")):
else:
# coding=utf-8
import os
from klein import Klein
from twisted.internet import reactor
from twisted.web.resource import Resource, ForbiddenResource
from twisted.web.server import Site
from twisted.web.static import File
from server.app import App
class FileNoDir(File):
def directoryListing(self):
return ForbiddenResource()
if __name__ == '__main__':
client_side_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'client_side'))
server = Klein()
app = App(server, client_side_dir)
root = server.resource()
site = Site(root)
"""
root.putChild("static", FileNoDir(os.path.join(client_side_dir, 'static')))
root.putChild("app", FileNoDir(os.path.join(client_side_dir, 'app')))
root.putChild("", File(os.path.join(client_side_dir, 'templates', 'index.html')))
root.putChild("templates", File(os.path.join(client_side_dir, 'templates')))
"""
reactor.listenTCP(8888, site)
reactor.run()
class ItemStore(object):
app = Klein()
def _init_(self):
self._items = {}
@app.route('/')
def items(self, request):
request.setHeader('Content-Type', 'application/json')
return json.dumps(self._items)
@app.route('/<string:name>', methods=['POST'])
def post_item(self, request, name):
request.setHeader('Content-Type', 'application/json')
content = json.loads(request.content.read())
print("123langCode")
print(content["langCode"])
langCode = content["langCode"]
text = content["data"]
print("pure text")
print(text)
text = text\
.replace("\r", " ")\
.replace("\n", " ") \
.replace("gm", " ") \
.replace("mg/dl", " ")\
.replace("high", " ") \
.lower()
text = re.sub(r'[^A-Za-z ]+', " is a ", text)
text = re.sub(' +', " ", text)
text = text \
.replace("high", " ")\
.replace("desirable", " ")\
.replace("borderline", " ")
text = re.sub(' +', " ", text)
print("regexed text")
print(text)
sentence = sp(text)
text = " ".join([token.lemma_ for token in sentence])
print("lemmatized text")
print(text)
doc = nlp(json.dumps(text))
entities = doc.ents
entitySet = set()
for entity in entities:
entitySet.add(str(entity))
splits = str(entity).split(" ")
for split in splits:
entitySet.add(str(split).strip())
print(entitySet)
return self.getQueryResults(entitySet, langCode)
def getQueryResults(self, entitySet, langCode):
data = {'entities': []}
for entity in entitySet:
# result_entity = dict()
result_entity = {"entityName": "", "comment": "", "foods": [], "diseases": []}
abstract_sparql_query = self.getAbstractQuery(entity, langCode)
abstract_query_results = abstract_sparql_query.query().convert()
print("abstract_query_results")
print(abstract_query_results)
if len(abstract_query_results["results"]["bindings"]) == 0:
continue
for result in abstract_query_results["results"]["bindings"]:
result_entity["entityName"] = entity
#result_entity['entityURI'] = str(result["chem"]["value"])
result_entity["comment"] = str(result["comment"]["value"])
#print(result_entity)
# food_sparql_query = self.getFoodQuery(entity, langCode)
# food_query_results = food_sparql_query.query().convert()
#
# for result in food_query_results["results"]["bindings"]:
# result_entity["foods"].append(str(result["label"]["value"]))
# food_dict = dict()
# food_sparql_query = self.getLowFoodQuery(entity, langCode)
# food_query_results = food_sparql_query.query().convert()
#
# for result in food_query_results["results"]["bindings"]:
# result_entity["foods"].append(str(result["label"]["value"]))
food_sparql_query = self.getFoodQuery(entity, langCode)
food_query_results = food_sparql_query.query().convert()
for result in food_query_results["results"]["bindings"]:
result_entity["foods"].append(str(result["label"]["value"]))
disease_sparql_query = self.getDiseaseQuery(entity, langCode)
disease_query_results = disease_sparql_query.query().convert()
for result in disease_query_results["results"]["bindings"]:
result_entity["diseases"].append(str(result["label"]["value"]))
data['entities'].append(result_entity)
return json.dumps(data)
def getAbstractQuery(self, entity, langCode):
sparql = SPARQLWrapper("http://dbpedia.org/sparql")
sparql.setReturnFormat(JSON)
query = """
PREFIX dbo: <http://dbpedia.org/ontology/>
PREFIX dbp: <http://dbpedia.org/resource/>
PREFIX foaf: <http://xmlns.com/foaf/0.1/>
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
SELECT distinct(?chem) ?comment where
{
?chem rdf:type dbo:ChemicalCompound .
?chem rdfs:label ?label .
?chem rdfs:comment ?comment .
FILTER regex(?label, "^%s$", "i")
FILTER (langMatches(lang(?comment),"%s"))
}
""" % (entity, langCode)
#print("query")
#print(query)
sparql.setQuery(query)
return sparql
def getFoodQuery(self, entity, langCode):
sparql = SPARQLWrapper("http://dbpedia.org/sparql")
sparql.setReturnFormat(JSON)
query = """
PREFIX dbo: <http://dbpedia.org/ontology/>
PREFIX dbp: <http://dbpedia.org/resource/>
PREFIX foaf: <http://xmlns.com/foaf/0.1/>
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
SELECT distinct(?food) ?label where
{
?food rdf:type dbo:Food .
?food rdfs:label ?label .
?food dbo:abstract ?abstract .
FILTER regex(?abstract, "%s", "i")
FILTER (langMatches(lang(?label),"%s"))
FILTER (langMatches(lang(?abstract),"%s"))
}
""" % (entity, langCode, langCode)
sparql.setQuery(query)
return sparql
def getLowFoodQuery(self, entity, langCode):
sparql = SPARQLWrapper("http://dbpedia.org/sparql")
sparql.setReturnFormat(JSON)
query = """
PREFIX dbo: <http://dbpedia.org/ontology/>
PREFIX dbp: <http://dbpedia.org/resource/>
PREFIX foaf: <http://xmlns.com/foaf/0.1/>
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
SELECT distinct(?food) ?label where
{
?food rdf:type dbo:Food .
?food rdfs:label ?label .
?food dbo:abstract ?abstract .
FILTER regex(?abstract, "%s", "i")
FILTER (langMatches(lang(?label),"%s"))
FILTER (langMatches(lang(?abstract),"%s"))
}
""" % (entity, langCode, langCode)
sparql.setQuery(query)
return sparql
def getDiseaseQuery(self, entity, langCode):
sparql = SPARQLWrapper("http://dbpedia.org/sparql")
sparql.setReturnFormat(JSON)
query = """
PREFIX dbo: <http://dbpedia.org/ontology/>
PREFIX dbp: <http://dbpedia.org/resource/>
PREFIX foaf: <http://xmlns.com/foaf/0.1/>
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
SELECT distinct(?chem) ?label where
{
?chem rdf:type dbo:Disease .
?chem rdfs:label ?label .
?chem dbo:abstract ?abstract .
FILTER regex(?abstract, "%s", "i")
FILTER (langMatches(lang(?label),"%s"))
FILTER (langMatches(lang(?abstract),"%s"))
}
""" % (entity, langCode, langCode)
sparql.setQuery(query)
return sparql
# -*- coding: utf-8 -*-
import mimetypes
import gc
from klein import Klein
from twisted.web.static import File
from basement import settings
from basement.utils import render_template
from service import service_registry
app = Klein()
@app.route("/")
def index(request):
return render_template('index.html')
@app.route("/bits/static/", branch=True)
def static(request):
return File(settings.STATIC_DIR)
@app.route('/<string:service>/<string:package>/<string:markwhat>/')
def badge_markwhat(request, service, package, markwhat):
gc.collect()
markwhat = markwhat.lower()
if markwhat not in settings.MARKWAHT_TEMPLATES:
request.setResponseCode(400)
klein_app = Klein()
import ConfigParser
from twisted.python import log
import sys
from leap.common.events import server as events_server
from pixelated.config import app_factory
import pixelated.config.args as input_args
import pixelated.bitmask_libraries.register as leap_register
from pixelated.bitmask_libraries.leap_srp import LeapAuthException
import pixelated.config.credentials_prompt as credentials_prompt
import pixelated.support.ext_protobuf # monkey patch for protobuf in OSX
import pixelated.support.ext_sqlcipher # monkey patch for sqlcipher in debian
app = Klein()
app.config = {}
def setup():
args = input_args.parse()
setup_debugger(args.debug)
if args.register:
register(*args.register[::-1])
else:
if args.dispatcher:
config = fetch_credentials_from_dispatcher(args.dispatcher)
app.config['LEAP_SERVER_NAME'] = config['leap_provider_hostname']
app.config['LEAP_USERNAME'] = config['user']
app.config['LEAP_PASSWORD'] = config['password']
import os
from klein import Klein
from twisted.web import server, static
from twisted.internet import reactor
app = Klein()
# staticFileDir = os.environ['HOME']
staticFileDir = './static'
@app.route('/static/', branch=True)
def staticFiles(request):
return static.File(staticFileDir)
app.run('localhost', 9000)
from klein import Klein
app = Klein()
@app.route('/user/<username>')
def pg_user(request, username):
return 'Hi %s!' % (username,)
@app.route('/user/bob')
def pg_user_bob(request):
return 'Hello there bob!'
app.run("localhost", 8080)
class BadlyBehavedHeaders(Headers):
"""
Make L{Headers} lie, and refuse to return a Host header from
getAllRequestHeaders.
"""
def getAllRawHeaders(self):
# type: () -> Iterable[Tuple[bytes, List[bytes]]]
"""
Don't return a host header.
"""
for key, values in super(BadlyBehavedHeaders, self).getAllRawHeaders():
if key != b'Host':
yield (key, values)
router = Klein()
requirer = Requirer()
@requirer.require(router.route("/hello/world", methods=['GET']),
url=RequestURL())
def requiresURL(url):
# type: (DecodedURL) -> Text
"""
This is a route that requires a URL.
"""
return url.child(u"hello/ world").asText()
class ISample(Interface):
"""
import treq
from klein import Klein
from twisted.internet.defer import inlineCallbacks, returnValue
from bs4 import BeautifulSoup
import json
import re
# from flask import jsonify
app = Klein()
comp_name_regex = re.compile(r'\b[\w ]+\b')
def clean_text(str, flag='td'):
if flag == 'td':
return str.find('td').get_text().replace('-', ' ').replace(
',', '').replace('.', '').strip().encode('utf-8', 'ignore')
elif flag == 'th':
return str.find('th').get_text().strip().replace('-', ' ').replace(
',', '').replace('.', '').split(':')[0].encode('utf-8', 'ignore')
@app.route('/')
def helloworld():
return "<h>Helloe!!<\h>"
@app.route('/apis/get', methods=[
'GET',
])
@inlineCallbacks
class RasaNLU(object):
"""Class representing Rasa NLU http server"""
app = Klein()
def __init__(self, config, component_builder=None, testing=False):
logging.basicConfig(filename=config['log_file'],
level=config['log_level'])
logging.captureWarnings(True)
logger.debug("Configuration: " + config.view())
logger.debug("Creating a new data router")
self.config = config
self.data_router = self._create_data_router(config, component_builder)
self._testing = testing
reactor.suggestThreadPoolSize(config['num_threads'] * 5)
def _create_data_router(self, config, component_builder):
return DataRouter(config, component_builder)
@app.route("/", methods=['GET', 'OPTIONS'])
@check_cors
def hello(self, request):
"""Main Rasa route to check if the server is online"""
return "hello from Rasa NLU: " + __version__
@app.route("/parse", methods=['GET', 'POST', 'OPTIONS'])
@requires_auth
@check_cors
@inlineCallbacks
def parse_get(self, request):
request.setHeader('Content-Type', 'application/json')
if request.method.decode('utf-8', 'strict') == 'GET':
request_params = {key.decode('utf-8', 'strict'): value[0].decode('utf-8', 'strict')
for key, value in request.args.items()}
else:
request_params = simplejson.loads(
request.content.read().decode('utf-8', 'strict'))
if 'query' in request_params:
request_params['q'] = request_params.pop('query')
if 'q' not in request_params:
request.setResponseCode(404)
dumped = json_to_string({"error": "Invalid parse parameter specified"})
returnValue(dumped)
else:
data = self.data_router.extract(request_params)
try:
request.setResponseCode(200)
response = yield (self.data_router.parse(data) if self._testing
else threads.deferToThread(self.data_router.parse, data))
returnValue(json_to_string(response))
except InvalidProjectError as e:
request.setResponseCode(404)
returnValue(json_to_string({"error": "{}".format(e)}))
except Exception as e:
request.setResponseCode(500)
logger.exception(e)
returnValue(json_to_string({"error": "{}".format(e)}))
@app.route("/version", methods=['GET', 'OPTIONS'])
@requires_auth
@check_cors
def version(self, request):
"""Returns the Rasa server's version"""
request.setHeader('Content-Type', 'application/json')
return json_to_string({'version': __version__})
@app.route("/config", methods=['GET', 'OPTIONS'])
@requires_auth
@check_cors
def rasaconfig(self, request):
"""Returns the in-memory configuration of the Rasa server"""
request.setHeader('Content-Type', 'application/json')
return json_to_string(self.config.as_dict())
@app.route("/status", methods=['GET', 'OPTIONS'])
@requires_auth
@check_cors
def status(self, request):
request.setHeader('Content-Type', 'application/json')
return json_to_string(self.data_router.get_status())
@app.route("/train", methods=['POST', 'OPTIONS'])
@requires_auth
@check_cors
@inlineCallbacks
def train(self, request):
data_string = request.content.read().decode('utf-8', 'strict')
kwargs = {key.decode('utf-8', 'strict'): value[0].decode('utf-8', 'strict')
for key, value in request.args.items()}
request.setHeader('Content-Type', 'application/json')
try:
request.setResponseCode(200)
response = yield self.data_router.start_train_process(
data_string, kwargs)
returnValue(json_to_string({'info': 'new model trained: {}'.format(response)}))
except AlreadyTrainingError as e:
request.setResponseCode(403)
returnValue(json_to_string({"error": "{}".format(e)}))
except InvalidProjectError as e:
request.setResponseCode(404)
returnValue(json_to_string({"error": "{}".format(e)}))
except TrainingException as e:
request.setResponseCode(500)
returnValue(json_to_string({"error": "{}".format(e)}))
@app.route("/evaluate", methods=['POST'])
@requires_auth
@check_cors
def evaluate(self, request):
data_string = request.content.read().decode('utf-8', 'strict')
params = {
key.decode('utf-8', 'strict'): value[0].decode('utf-8', 'strict')
for key, value in request.args.items()
}
request.setHeader('Content-Type', 'application/json')
try:
request.setResponseCode(200)
response = self.data_router.evaluate(data_string,
params.get('project'),
params.get('model'))
return simplejson.dumps(response)
except Exception as e:
request.setResponseCode(500)
return simplejson.dumps({"error": "{}".format(e)})
# -*- coding: utf-8 -*-
from __future__ import print_function, unicode_literals
# Cobble together a deterministic random function using a string as a seed.
from random import Random
from hashlib import sha256
from struct import unpack
def random_from_string(string):
return Random(
unpack("!I", sha256(string.encode("utf-8")).digest()[:4])[0]
)
from twisted.web.template import tags, slot
from klein import Klein, Plating
app = Klein()
myStyle = Plating(
tags=tags.html(
tags.head(tags.title(slot("pageTitle"))),
tags.body(tags.h1(slot("pageTitle"), Class="titleHeading"),
tags.div(slot(Plating.CONTENT)))
),
defaults={"pageTitle": "Places & Foods"}
)
@myStyle.routed(
app.route("/"),
tags.div(
tags.h2("Sample Places:"),
tags.ul([tags.li(tags.a(href=["/places/", place])(place))
for place in ["new york", "san francisco", "shanghai"]]),
import pickle
from klein import Klein
import json
from twisted.internet import defer
app = Klein()
class Order(object):
def __getstate__(self):
return self.json_dict()
def __init__(self, order_id):
self.order_id = order_id
self.mid = None
self.tid = None
self.amount = None
self.payment = None
def json_dict(self):
return {
'order_id': self.order_id,
'mid': self.mid,
'tid': self.tid,
'amount': self.amount,
'payment': self.payment
}
class Orders(object):
def __init__(self):
try:
f = open('/tmp/orders.pickle', 'rb')
class WebInterface(YomboLibrary, BuildDistribution, ErrorHandler, Render,
WebServer):
"""
Web interface framework.
"""
webapp = Klein() # Like Flask, but for twisted
visits = 0
starting = True
already_starting_web_servers = False
hook_listeners = {} # special way to toss hook calls to routes.
def _init_(self, **kwargs):
self.frontend_building = False
self.web_interface_fully_started = False
self.enabled = self._Configs.get("webinterface", "enabled", True)
self.fqdn = self._Configs.get2("dns", "fqdn", None, False)
self.enabled = self._Configs.get("core", "enabled", True)
if not self.enabled:
return
self.file_cache = ExpiringDict(
max_len=100, max_age_seconds=120
) # used to load a few static files into memory that are commonly used.
self.translators = {}
self.idempotence = self._Cache.ttl(name="lib.webinterface.idempotence",
ttl=300)
self.working_dir = self._Atoms.get("working_dir")
self.app_dir = self._Atoms.get("app_dir")
self.wi_dir = "/lib/webinterface"
self.misc_wi_data = {}
self.wi_port_nonsecure = self._Configs.get2("webinterface",
"nonsecure_port", 8080)
self.wi_port_secure = self._Configs.get2("webinterface", "secure_port",
8443)
self.webapp.templates = jinja2.Environment(
loader=jinja2.FileSystemLoader(f"{self.app_dir}/yombo"),
extensions=["jinja2.ext.loopcontrols"])
self.setup_basic_filters()
self.web_interface_listener = None
self.web_interface_ssl_listener = None
self.api_stream_spectators = {}
# Load API routes
route_api_v1_atoms(self.webapp)
route_api_v1_automation_rules(self.webapp)
# route_api_v1_camera(self.webapp)
route_api_v1_debug(self.webapp)
route_api_v1_device(self.webapp)
route_api_v1_device_command(self.webapp)
# route_api_v1_events(self.webapp)
# route_api_v1_gateway(self.webapp)
# route_api_v1_module(self.webapp)
route_api_v1_mqtt(self.webapp)
# route_api_v1_notification(self.webapp)
route_api_v1_scenes(self.webapp)
# route_api_v1_server(self.webapp)
# route_api_v1_statistics(self.webapp)
# route_api_v1_stream(self.webapp, self)
route_api_v1_states(self.webapp)
route_api_v1_system(self.webapp)
# route_api_v1_storage(self.webapp)
route_api_v1_user(self.webapp)
# route_api_v1_webinterface_logs(self.webapp)
# Load web server routes
route_home(self.webapp)
route_misc(self.webapp)
route_system(self.webapp)
route_user(self.webapp)
if self.operating_mode != "run":
from yombo.lib.webinterface.routes.restore import route_restore
from yombo.lib.webinterface.routes.setup_wizard import route_setup_wizard
route_setup_wizard(self.webapp)
route_restore(self.webapp)
self.npm_build_results = None
self.temp_data = ExpiringDict(max_age_seconds=1800)
self.web_server_started = False
self.web_server_ssl_started = False
self.web_factory = None
self.user_login_tokens = self._Cache.ttl(name="lib.users.cache",
ttl=300)
@property
def operating_mode(self):
return self._Loader.operating_mode
def _load_(self, **kwargs):
if not self.enabled:
return
if self.operating_mode == "run":
self.build_dist() # Make all the JS and CSS files
self.module_config_links = {}
# self.web_factory = Yombo_Site(self.webapp.resource(), None, logPath="/dev/null")
self.web_factory = Yombo_Site(self.webapp.resource(),
None,
logPath=None)
self.web_factory.setup_log_queue(self)
self.web_factory.noisy = False # turn off Starting/stopping message
self.displayTracebacks = False
self._display_how_to_access_at = 0 # When the display notice for how to access the web was shown.
self.misc_wi_data["gateway_label"] = self._Configs.get2(
"core", "label", "Yombo Gateway", False)
self.misc_wi_data["operating_mode"] = self.operating_mode
self.misc_wi_data["notifications"] = self._Notifications
self.misc_wi_data[
"notification_priority_map_css"] = NOTIFICATION_PRIORITY_MAP_CSS
self.misc_wi_data["breadcrumb"] = []
self.webapp.templates.globals["yombo"] = self
self.webapp.templates.globals["_local_gateway"] = self._Gateways.local
self.webapp.templates.globals["_amqp"] = self._AMQP
self.webapp.templates.globals["_amqpyombo"] = self._AMQPYombo
self.webapp.templates.globals["_authkeys"] = self._AuthKeys
self.webapp.templates.globals["_atoms"] = self._Atoms
self.webapp.templates.globals["_automation"] = self._Automation
self.webapp.templates.globals["_cache"] = self._Cache
self.webapp.templates.globals["_calllater"] = self._CallLater
self.webapp.templates.globals["_commands"] = self._Commands
self.webapp.templates.globals["_configs"] = self._Configs
self.webapp.templates.globals["_crontab"] = self._CronTab
self.webapp.templates.globals["_events"] = self._Events
self.webapp.templates.globals["_devices"] = self._Devices
self.webapp.templates.globals["_devicetypes"] = self._DeviceTypes
self.webapp.templates.globals[
"_downloadmodules"] = self._DownloadModules
self.webapp.templates.globals["_gatewaycoms"] = self._GatewayComs
self.webapp.templates.globals["_gateways"] = self._Gateways
self.webapp.templates.globals["_gpg"] = self._GPG
self.webapp.templates.globals["_inputtypes"] = self._InputTypes
self.webapp.templates.globals["_intents"] = self._Intents
self.webapp.templates.globals["_localize"] = self._Localize
self.webapp.templates.globals["_locations"] = self._Locations
self.webapp.templates.globals["_locations"] = self._Locations
self.webapp.templates.globals["_modules"] = self._Modules
self.webapp.templates.globals["_mqtt"] = self._MQTT
self.webapp.templates.globals["_nodes"] = self._Nodes
self.webapp.templates.globals["_notifiticaions"] = self._Notifications
self.webapp.templates.globals["_users"] = self._Users
self.webapp.templates.globals["_queue"] = self._Queue
self.webapp.templates.globals["_scenes"] = self._Scenes
self.webapp.templates.globals["_requests"] = self._Requests
self.webapp.templates.globals["_sqldict"] = self._SQLDict
self.webapp.templates.globals["_sslcerts"] = self._SSLCerts
self.webapp.templates.globals["_states"] = self._States
self.webapp.templates.globals["_statistics"] = self._Statistics
self.webapp.templates.globals["_storage"] = self._Storage
self.webapp.templates.globals["_tasks"] = self._Tasks
self.webapp.templates.globals["_times"] = self._Times
self.webapp.templates.globals["_variabledata"] = self._VariableData
self.webapp.templates.globals["_variablefields"] = self._VariableFields
self.webapp.templates.globals["_variablegroups"] = self._VariableGroups
self.webapp.templates.globals["_validate"] = self._Validate
self.webapp.templates.globals["_webinterface"] = self
self.webapp.templates.globals["py_randint"] = randint
self.webapp.templates.globals["py_time_time"] = time
self.webapp.templates.globals["py_urllib_urlparse"] = urlparse
self.webapp.templates.globals["py_urllib_urlunparse"] = urlunparse
self.webapp.templates.globals["yombo_utils"] = yombo.utils
self.webapp.templates.globals["misc_wi_data"] = self.misc_wi_data
self.webapp.templates.globals["webinterface"] = self
self.webapp.templates.globals["_location_id"] = None
self.webapp.templates.globals["_area_id"] = None
self.webapp.templates.globals["_location"] = None
self.webapp.templates.globals["_area"] = None
self.webapp.templates.globals["bg_image_id"] = lambda: int(time() / 300
) % 6
self.webapp.templates.globals["get_alerts"] = self.get_alerts
self._refresh_jinja2_globals_()
self.starting = False
self.start_web_servers()
self.clean_idempotence_ids_loop = LoopingCall(
self.clean_idempotence_ids)
self.clean_idempotence_ids_loop.start(1806, False)
def _refresh_jinja2_globals_(self, **kwargs):
"""
Update various globals for the Jinja2 template.
:return:
"""
if self.operating_mode != "run":
return
self.webapp.templates.globals[
"_location_id"] = self._Locations.location_id
self.webapp.templates.globals["_area_id"] = self._Locations.area_id
self.webapp.templates.globals["_location"] = self._Locations.location
self.webapp.templates.globals["_area"] = self._Locations.area
@inlineCallbacks
def _start_(self, **kwargs):
self._Notifications.add({
"title": "System still starting",
"message": "Still starting up. Please wait.",
"source": "Web Interface Library",
"persist": True,
"priority": "high",
"always_show": True,
"always_show_allow_clear": False,
"id": "webinterface:starting",
})
if self.operating_mode != "run":
logger.warn(
"First time running the gateway, it will take a while to build the frontend web server."
)
yield self.build_dist(
verbose=True) # Make all the JS and CSS files
self._get_nav_side_items()
self.webapp.templates.globals["_"] = _ # i18n
def _started_(self, **kwargs):
# if self.operating_mode != "run":
self._display_how_to_access_at = int(time())
self.display_how_to_access()
self._Notifications.delete("webinterface:starting")
self.web_interface_fully_started = True
self.send_hook_listeners_ping_loop = LoopingCall(
self.send_hook_listeners_ping_loop)
self.send_hook_listeners_ping_loop.start(55, True)
def clean_idempotence_ids(self):
"""
Removes older idempotence keys.
:return:
"""
delete_time = int(time()) - 1800
for key in self.idempotence.keys():
if self.idempotence[key] < delete_time:
del self.idempotence[key]
def send_hook_listeners_ping_loop(self):
route_api_v1_stream_broadcast(self, "ping", int(time()))
def register_hook(self, name, thecallback):
if name not in self.hook_listeners:
self.hook_listeners[name] = []
self.hook_listeners[name].append(thecallback)
@inlineCallbacks
def _yombo_universal_hook_(self, hook_name=None, **kwargs):
"""
Implements the universal hook.
:param kwargs:
:return:
"""
if hook_name in self.hook_listeners:
for a_callback in self.hook_listeners[hook_name]:
d = Deferred()
d.addCallback(lambda ignored: maybeDeferred(
a_callback, self, hook_name=hook_name, **kwargs))
d.addErrback(self.yombo_universal_hook_failure, hook_name,
a_callback)
d.callback(1)
yield d
def yombo_universal_hook_failure(self, failure, hook_name, acallback):
logger.warn(
"---==(failure WI:universal hook for hook ({hook_name})==----",
hook_name=hook_name)
logger.warn("--------------------------------------------------------")
logger.warn("{acallback}", acallback=acallback)
logger.warn("{failure}", failure=failure)
logger.warn("--------------------------------------------------------")
raise RuntimeError(f"failure during module invoke for hook: {failure}")
def check_have_required_nodes(self):
try:
node = yield self._Nodes.get("main_page", "webinterface_page")
except KeyError as e:
pass
# add base node...
def _configuration_set_(self, **kwargs):
"""
Need to monitor if the web interface port has changed. This will restart the webinterface
server if needed.
:param kwargs: section, option(key), value
:return:
"""
section = kwargs["section"]
option = kwargs["option"]
value = kwargs["value"]
if self.starting is True:
return
if section == "webinterface":
if option == "nonsecure_port":
self.change_ports(port_nonsecure=value)
elif option == "secure_port":
self.change_ports(port_secure=value)
@inlineCallbacks
def _unload_(self, **kwargs):
if hasattr(self, "web_factory"):
if self.web_factory is not None:
yield self.web_factory.save_log_queue()
@property
def internal_url(self):
"""
Returns the starting portion of the URL to this host.
https://i.exmaple.yombo.net
:return:
"""
fqdn = self.fqdn()
if fqdn is None:
internal_hostname = self._Configs.get("core", "localipaddress_v4")
return f"http://{internal_hostname}:{self.wi_port_nonsecure()}"
else:
return f"https://i.{fqdn}:{self.wi_port_secure()}"
@property
def external_url(self):
"""
Returns the starting portion of the URL to this host.
https://e.exmaple.yombo.net
:return:
"""
fqdn = self.fqdn()
if fqdn is None:
external_hostname = self._Configs.get("core",
"externalipaddress_v4")
return f"https://{external_hostname}:{self.wi_port_secure()}"
else:
return f"https://e.{fqdn}:{self.wi_port_secure()}"
def i18n(self, request):
"""
Gets a translator based on the language the browser provides us.
:param request: The browser request.
:return:
"""
locales = self._Localize.parse_accept_language(
request.getHeader("accept-language"))
locales_hash = yombo.utils.sha256_compact("".join(
str(e) for e in locales))
if locales_hash in self.translators:
return self.translators[locales_hash]
else:
self.translators[locales_hash] = web_translator(self, locales)
return self.translators[locales_hash]
@inlineCallbacks
def _get_nav_side_items(self, **kwargs):
"""
Called before modules have their _prestart_ function called (after _load_).
This implements the hook "webinterface_add_routes" and calls all libraries and modules. It allows libs and
modules to add menus to the web interface and provide additional funcationality.
**Usage**:
.. code-block:: python
def ModuleName_webinterface_add_routes(self, **kwargs):
return {
"nav_side": [
{
"label1": "Tools",
"label2": "MQTT",
"priority1": 3000,
"priority2": 10000,
"icon": "fa fa-wrench fa-fw",
"url": "/tools/mqtt",
"tooltip": "",
"opmode": "run",
"cluster": "any",
},
],
"routes": [
self.web_interface_routes,
],
"configs" {
"settings_link": "/modules/tester/index",
},
}
"""
# first, lets get the top levels already defined so children don"t re-arrange ours.
top_levels = {}
add_on_menus = yield global_invoke_all(
"_webinterface_add_routes_",
called_by=self,
)
logger.debug("_webinterface_add_routes_ results: {add_on_menus}",
add_on_menus=add_on_menus)
nav_side_menu = deepcopy(NAV_SIDE_MENU)
for component, options in add_on_menus.items():
if "nav_side" in options:
for new_menu in options["nav_side"]:
if new_menu["label1"] in nav_side_menu:
the_index = nav_side_menu[new_menu["label1"]].index(
"bar")
new_menu["priority1"] = nav_side_menu[the_index][
"priority1"]
if "priority1" not in new_menu or isinstance(
new_menu["priority1"], int) is False:
new_menu["priority1"] = 1000
if "priority2" not in new_menu or isinstance(
new_menu["priority2"], int) is False:
new_menu["priority2"] = 100
nav_side_menu = nav_side_menu + options["nav_side"]
temp_list = sorted(NAV_SIDE_MENU,
key=itemgetter("priority1", "label1", "priority2"))
for item in temp_list:
label1 = item["label1"]
if label1 not in temp_list:
top_levels[label1] = item["priority1"]
for component, options in add_on_menus.items():
logger.debug("component: {component}, options: {options}",
component=component,
options=options)
if "menu_priorities" in options: # allow modules to change the ordering of top level menus
for label, priority in options["menu_priorities"].items():
top_levels[label] = priority
if "routes" in options:
for new_route in options["routes"]:
new_route(self.webapp)
if "configs" in options:
if "settings_link" in options["configs"]:
self.module_config_links[component._module_id] = options[
"configs"]["settings_link"]
# build menu tree
self.misc_wi_data["nav_side"] = {}
is_master = self.is_master
# temp_list = sorted(nav_side_menu, key=itemgetter("priority1", "priority2", "label1"))
temp_list = sorted(nav_side_menu,
key=itemgetter("priority1", "label1", "priority2",
"label2"))
for item in temp_list:
if "cluster" not in item:
item["cluster"] = "any"
if item["cluster"] == "master" and is_master is not True:
continue
if item["cluster"] == "member" and is_master is True:
continue
item["label1_text"] = deepcopy(item["label1"])
item["label2_text"] = deepcopy(item["label2"])
label1 = "ui::navigation::" + yombo.utils.snake_case(
item["label1"])
item["label1"] = "ui::navigation::" + yombo.utils.snake_case(
item["label1"])
item["label2"] = "ui::navigation::" + yombo.utils.snake_case(
item["label2"])
if label1 not in self.misc_wi_data["nav_side"]:
self.misc_wi_data["nav_side"][label1] = []
self.misc_wi_data["nav_side"][label1].append(item)
self.starting = False
def add_alert(self,
session,
message,
level="info",
display_once=True,
deletable=True,
id=None):
"""
Add an alert to the stack.
:param level: info, warning, error
:param message:
:return:
"""
id = session.add_alert(message, level, display_once, deletable, id)
return id
def get_alerts(self, session, autodelete=None):
"""
Retrieve a list of alerts for display.
"""
if session is None:
return {}
return session.get_alerts(autodelete)
def get_template(self, request, template_path):
request.webinterface.webapp.templates.globals[
"_"] = request.webinterface.i18n(
request) # set in auth.update_request.
return self.webapp.templates.get_template(template_path)
def redirect(self, request, redirect_path):
request.redirect(redirect_path)
def _get_parms(self, request):
return parse_qs(urlparse(request.uri).query)
def request_get_default(self, request, name, default, offset=None):
if offset == None:
offset = 0
try:
return request.args.get(name)[offset]
except:
return default
def home_breadcrumb(self, request):
self.add_breadcrumb(request, "/", "Home")
def add_breadcrumb(self,
request,
url=None,
text=None,
show=None,
style=None,
data=None):
if hasattr(request, "breadcrumb") is False:
request.breadcrumb = []
self.misc_wi_data["breadcrumb"] = request.breadcrumb
if show is None:
show = True
if style is None:
style = "link"
elif style == "select_groups":
items = {}
for option_label, option_data in data.items():
items[option_label] = []
for select_text, select_url in option_data.items():
selected = ""
option_style = "None"
if select_url.startswith("$"):
selected = "selected"
select_url = select_url[1:]
elif select_url.startswith("#"):
option_style = "divider"
items[option_label].append({
"option_style": option_style,
"text": select_text,
"url": select_url,
"selected": selected,
})
data = items
elif style == "select":
items = []
for select_text, select_url in data.items():
selected = ""
option_style = "None"
if select_url.startswith("$"):
selected = "selected"
select_url = select_url[1:]
elif select_url.startswith("#"):
option_style = "divider"
items.append({
"option_style": option_style,
"text": select_text,
"url": select_url,
"selected": selected,
})
data = items
hash = sha256(
str(
str(url) + str(text) + str(show) + str(style) +
json.dumps(data)).encode()).hexdigest()
breadcrumb = {
"hash": hash,
"url": url,
"text": text,
"show": show,
"style": style,
"data": data,
}
request.breadcrumb.append(breadcrumb)
def setup_basic_filters(self):
self.webapp.templates.filters["yes_no"] = yombo.utils.is_yes_no
self.webapp.templates.filters["true_false"] = yombo.utils.is_true_false
self.webapp.templates.filters["excerpt"] = yombo.utils.excerpt
self.webapp.templates.filters["make_link"] = yombo.utils.make_link
self.webapp.templates.filters[
"status_to_string"] = converters.status_to_string
self.webapp.templates.filters[
"public_to_string"] = converters.public_to_string
self.webapp.templates.filters[
"epoch_to_string"] = converters.epoch_to_string
self.webapp.templates.filters[
"epoch_get_age"] = dt_util.get_age # yesterday, 5 minutes ago, etc.
self.webapp.templates.filters[
"epoch_get_age_exact"] = dt_util.get_age_exact # yesterday, 5 minutes ago, etc.
self.webapp.templates.filters[
"format_markdown"] = yombo.utils.format_markdown
self.webapp.templates.filters[
"hide_none"] = yombo.utils.display_hide_none
self.webapp.templates.filters[
"display_encrypted"] = self._GPG.display_encrypted
self.webapp.templates.filters[
"display_temperature"] = self._Localize.display_temperature
self.webapp.templates.filters["json_human"] = yombo.utils.json_human
self.webapp.templates.filters["yombo"] = self
def restart(self, request, message=None, redirect=None):
if message is None:
message = ""
if redirect is None:
redirect = "/"
page = self.get_template(request, self.wi_dir + "/pages/restart.html")
reactor.callLater(0.3, self.do_restart)
return page.render(message=message,
redirect=redirect,
uptime=str(self._Atoms["running_since"]))
def do_restart(self):
try:
raise YomboRestart("Web Interface setup wizard complete.")
except:
pass
def shutdown(self, request):
page = self.get_template(request, self.wi_dir + "/pages/shutdown.html")
# reactor.callLater(0.3, self.do_shutdown)
return page.render()
def do_shutdown(self):
raise YomboCritical("Web Interface setup wizard complete.")
import time
from klein import Klein
from redis import Redis
import requests
from yarg.package import json2package
PYPI_URL = "https://pypi.python.org/pypi/%s/json"
SHIELD_URL = "http://img.shields.io/badge/%s-%s-%s.%s"
# SHIELD_URL = "http://localhost:9000/badge/%s-%s-%s.%s" # pypip.in uses a local version of img.shields.io
FILE_CACHE = "/tmp/shields.py/"
CACHE_TIME = (60 * 60) * 24 # 24 hours
REDIS_EXPIRE = 60 * 10 # 10 minutes
app = Klein()
redis = Redis()
def format_number(singular, number):
value = singular % {'value': number}
# Get rid of the .0 but keep the other decimals
return value.replace('.0', '')
intword_converters = (
(3, lambda number: format_number('%(value).1fk', number)),
(6, lambda number: format_number('%(value).1fM', number)),
(9, lambda number: format_number('%(value).1fB', number)),
)
args = vars(parser.parse_args())
PORT = args["port"]
ADDRESS = args["ip"]
LOG = args["log"]
print(f"Connecting on {ADDRESS} on {PORT}")
"""
Starting klein http-server
"""
from klein import Klein
from twisted.internet.defer import succeed
import wave
app = Klein()
@app.route("/transcribe", methods=["POST"])
def transcribe(request):
fname = f"{str(time.time()).replace('.','-')}.wav"
cvtfname = "cvt" + fname
with open(fname, "wb") as fileOutput:
fileOutput.write(request.content.read())
"""
Currently, only 16-bit, 16 kHz, mono-channel WAVE audio files are supported in the Python client
"""
converted = False
def setUp(self):
"""
Create an app and a resource wrapping that app for this test.
"""
self.app = Klein()
self.kr = self.app.resource()
