Beispiel #1
0
    def test_resource(self, mock_kr):
        """
        L{Klien.resource} returns a L{KleinResource}.
        """
        app = Klein()
        resource = app.resource()

        mock_kr.assert_called_with(app)
        self.assertEqual(mock_kr.return_value, resource)
Beispiel #2
0
 def test_runTCP6(self, reactor, mock_sfs, mock_log, mock_kr):
     """
     L{Klein.run} called with tcp6 endpoint description.
     """
     app = Klein()
     interface = "2001\:0DB8\:f00e\:eb00\:\:1"
     spec = "tcp6:8080:interface={0}".format(interface)
     app.run(endpoint_description=spec)
     reactor.run.assert_called_with()
     mock_sfs.assert_called_with(reactor, spec)
     mock_log.startLogging.assert_called_with(sys.stdout)
     mock_kr.assert_called_with(app)
Beispiel #3
0
    def test_private_not_visible(self):
        """
        When an endpoint is decorated with ``@private_api`` it is
        omitted from result of ``makeRst``.
        """
        app = Klein()

        @private_api
        def g():
            pass

        app.route(b"/g", methods=[b"GET"])(g)
        rest = list(makeRst(b"/", "section", app, None, {}))
        self.assertEqual(rest, [])
Beispiel #4
0
 def test_runSSL(self, reactor, mock_sfs, mock_log, mock_kr):
     """
     L{Klein.run} called with SSL endpoint specification.
     """
     app = Klein()
     key = "key.pem"
     cert = "cert.pem"
     dh_params = "dhparam.pem"
     spec_template = "ssl:443:privateKey={0}:certKey={1}"
     spec = spec_template.format(key, cert, dh_params)
     app.run(endpoint_description=spec)
     reactor.run.assert_called_with()
     mock_sfs.assert_called_with(reactor, spec)
     mock_log.startLogging.assert_called_with(sys.stdout)
     mock_kr.assert_called_with(app)
Beispiel #5
0
    def test_error_handlers_list_is_copied(self):
        """
        L{Klein.__copy__} returns a new L{Klein} with all the error handlers
        """
        app = Klein()

        app.handle_errors(ValueError)(lambda request, failure: 'foo')

        app_copy = copy.copy(app)

        self.assertEquals(app._error_handlers, app_copy._error_handlers)

        app.handle_errors(KeyError)(lambda request, failure: 'foo')

        self.assertNotEquals(app._error_handlers, app_copy._error_handlers)
Beispiel #6
0
    def test_route(self):
        """
        L{Klein.route} adds functions as routable endpoints.
        """
        app = Klein()

        @app.route("/foo")
        def foo(request):
            return "foo"

        c = app.url_map.bind("foo")
        self.assertEqual(c.match("/foo"), ("foo", {}))
        self.assertEqual(len(app.endpoints), 1)

        self.assertEqual(app.execute_endpoint("foo", DummyRequest(1)), "foo")
Beispiel #7
0
    def test_runWithLogFile(self, reactor, mock_log, mock_site, mock_kr):
        """
        L{Klein.run} logs to the specified C{logFile}.
        """
        app = Klein()

        logFile = Mock()
        app.run("localhost", 8080, logFile=logFile)

        reactor.listenTCP.assert_called_with(8080, mock_site.return_value, interface="localhost")

        reactor.run.assert_called_with()

        mock_site.assert_called_with(mock_kr.return_value)
        mock_kr.assert_called_with(app)
        mock_log.startLogging.assert_called_with(logFile)
Beispiel #8
0
    def test_run(self, reactor, mock_log, mock_site, mock_kr):
        """
        L{Klein.run} configures a L{KleinResource} and a L{Site}
        listening on the specified interface and port, and logs
        to stdout.
        """
        app = Klein()

        app.run("localhost", 8080)

        reactor.listenTCP.assert_called_with(8080, mock_site.return_value, interface="localhost")

        reactor.run.assert_called_with()

        mock_site.assert_called_with(mock_kr.return_value)
        mock_kr.assert_called_with(app)
        mock_log.startLogging.assert_called_with(sys.stdout)
Beispiel #9
0
def webserver_for_test(test, url_path, response_content):
    """
    Create a webserver that serves ``response_content`` from ``url_path``.
    """
    app = Klein()

    @app.route(url_path)
    def _respond(request):
        return response_content
    factory = Site(app.resource())
    endpoint = serverFromString(reactor, b"tcp:0")
    listening = endpoint.listen(factory)

    def stop_port(port):
        test.addCleanup(port.stopListening)
        return port
    listening.addCallback(stop_port)
    return listening
Beispiel #10
0
    def test_submountedRoute(self):
        """
        L{Klein.subroute} adds functions as routable endpoints.
        """
        app = Klein()

        with app.subroute("/sub") as app:
            @app.route("/prefixed_uri")
            def foo_endpoint(request):
                return b"foo"

        c = app.url_map.bind("sub/prefixed_uri")
        self.assertEqual(
            c.match("/sub/prefixed_uri"), ("foo_endpoint", {}))
        self.assertEqual(
            len(app.endpoints), 1)
        self.assertEqual(
            app.execute_endpoint("foo_endpoint", DummyRequest(1)), b"foo")
Beispiel #11
0
    def test_stackedRoute(self):
        """
        L{Klein.route} can be stacked to create multiple endpoints of
        a single function.
        """
        app = Klein()

        @app.route("/foo")
        @app.route("/bar", endpoint="bar")
        def foobar(request):
            return "foobar"

        self.assertEqual(len(app.endpoints), 2)

        c = app.url_map.bind("foo")
        self.assertEqual(c.match("/foo"), ("foobar", {}))
        self.assertEqual(app.execute_endpoint("foobar", DummyRequest(1)), "foobar")

        self.assertEqual(c.match("/bar"), ("bar", {}))
        self.assertEqual(app.execute_endpoint("bar", DummyRequest(2)), "foobar")
Beispiel #12
0
    def test_routes(self):
        """
        L{getRoutes} returns all the defined routes and their attributes.
        """
        app = Klein()

        def f():
            pass
        f.attr = "attr"

        def g():
            pass
        g.attr = "G"
        app.route(b"/", methods=[b"GET"])(f)
        app.route(b"/g", methods=[b"PUT", b"OPTIONS"])(g)

        routes = sorted(getRoutes(app))
        self.assertEqual(routes, [
            KleinRoute(methods={b"GET"}, path=b"/", endpoint="f",
                       attributes={'attr': 'attr'}),
            KleinRoute(methods={b'PUT', b'OPTIONS'}, path=b'/g', endpoint='g',
                       attributes={'attr': 'G'})])
Beispiel #13
0
    def test_copy(self):
        """
        L{Klein.__copy__} returns a new L{Klein} with all the registered endpoints
        """
        app = Klein()

        @app.route("/foo")
        def foo(request):
            return "foo"

        app_copy = copy.copy(app)

        @app.route('/bar')
        def bar(request):
            return 'bar'

        dr1 = DummyRequest(1)
        dr2 = DummyRequest(2)
        dr3 = DummyRequest(3)

        self.assertEquals(app.execute_endpoint('foo', dr1), 'foo')
        self.assertEquals(app.execute_endpoint('bar', dr2), 'bar')
        self.assertRaises(KeyError, app_copy.execute_endpoint, 'bar', dr3)
Beispiel #14
0
def main(reactor):
    component = Component(
        transports=u"ws://localhost:8080/ws",
        realm=u"crossbardemo",
    )
    app = Klein()
    webapp = WebApplication(app, component)

    # have our Web site listen on 8090
    site = Site(app.resource())
    server_ep = TCP4ServerEndpoint(reactor, 8090)
    port = yield server_ep.listen(site)
    print("Web application on {}".format(port))

    # we don't *have* to hand over control of the reactor to
    # component.run -- if we don't want to, we call .start()
    # The Deferred it returns fires when the component is "completed"
    # (or errbacks on any problems).
    comp_d = component.start(reactor)

    # When not using run() we also must start logging ourselves.
    import txaio
    txaio.start_logging(level='info')

    # If the Component raises an exception we want to exit. Note that
    # things like failing to connect will be swallowed by the
    # re-connection mechanisms already so won't reach here.

    def _failed(f):
        print("Component failed: {}".format(f))
        done.errback(f)
    comp_d.addErrback(_failed)

    # wait forever (unless the Component raises an error)
    done = Deferred()
    yield done
Beispiel #15
0
import cgi
from urlparse import parse_qsl

from klein import Klein
from twisted.python.components import registerAdapter
from twisted.internet import reactor
from twisted.web.server import Session
from twisted.web.static import File
from zope.interface import Interface, Attribute, implements
from jinja2 import Environment, FileSystemLoader

from mailclient import sendmail
from imap import get_messages

app = Klein()
env = Environment(loader=FileSystemLoader('templates'))

def render_template(filename, **kwargs):
    template = env.get_template(filename)
    return template.render(**kwargs)

class ILogin(Interface):
    email = Attribute("user's email address")
    password = Attribute("user's email password")

class Login(object):
    implements(ILogin)
    def __init__(self, session):
        self.email = ''
        self.password = ''
Beispiel #16
0
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
###############################################################################

from twisted.internet.defer import inlineCallbacks, returnValue
from klein import Klein
from autobahn.twisted.wamp import Application

app = Klein()
wampapp = Application()


@app.route('/square/submit', methods=['POST'])
@inlineCallbacks
def square_submit(request):
    x = int(request.args.get('x', [0])[0])
    res = yield wampapp.session.call(u'com.example.square', x)
    returnValue("{} squared is {}".format(x, res))


if __name__ == "__main__":
    import sys
    from twisted.python import log
    from twisted.web.server import Site
Beispiel #17
0
 class _Another(object):
     anotherKlein = Klein()
     @anotherKlein.route("/bar")
     def bar(self):
         pass
Beispiel #18
0
class RasaCoreServer(object):
    """Class representing a Rasa Core HTTP server."""

    app = Klein()

    def __init__(self,
                 model_directory,
                 interpreter=None,
                 loglevel="INFO",
                 logfile="rasa_core.log",
                 cors_origins=None,
                 action_factory=None,
                 auth_token=None,
                 tracker_store=None):

        _configure_logging(loglevel, logfile)

        self.config = {
            "cors_origins": cors_origins if cors_origins else [],
            "token": auth_token
        }
        self.model_directory = model_directory
        self.interpreter = interpreter
        self.tracker_store = tracker_store
        self.action_factory = action_factory
        self.agent = self._create_agent(model_directory, interpreter,
                                        action_factory, tracker_store)

    @staticmethod
    def _create_agent(
        model_directory,  # type: Text
        interpreter,  # type: Union[Text, NaturalLanguageInterpreter]
        action_factory=None,  # type: Optional[Text]
        tracker_store=None  # type: Optional[TrackerStore]
    ):
        # type: (...) -> Optional[Agent]
        try:

            return Agent.load(model_directory,
                              interpreter,
                              tracker_store=tracker_store,
                              action_factory=action_factory)
        except Exception as e:
            logger.warn("Failed to load any agent model. Running "
                        "Rasa Core server with out loaded model now. {}"
                        "".format(e))
            return None

    @app.route("/", methods=['GET', 'OPTIONS'])
    @check_cors
    def hello(self, request):
        """Check if the server is running and responds with the version."""
        return "hello from Rasa Core: " + __version__

    @app.route("/conversations/<sender_id>/continue",
               methods=['POST', 'OPTIONS'])
    @check_cors
    @requires_auth
    @ensure_loaded_agent
    def continue_predicting(self, request, sender_id):
        """Continue a prediction started with parse.

        Caller should have executed the action returned from the parse
        endpoint. The events returned from that executed action are
        passed to continue which will trigger the next action prediction.

        If continue predicts action listen, the caller should wait for the
        next user message."""

        request.setHeader('Content-Type', 'application/json')
        request_params = json.loads(request.content.read().decode(
            'utf-8', 'strict'))
        encoded_events = request_params.get("events", [])
        executed_action = request_params.get("executed_action", None)
        evts = events.deserialise_events(encoded_events)
        try:
            response = self.agent.continue_message_handling(
                sender_id, executed_action, evts)
        except ValueError as e:
            request.setResponseCode(400)
            return json.dumps({"error": e.message})
        except Exception as e:
            request.setResponseCode(500)
            logger.exception(e)
            return json.dumps({"error": "Server failure. Error: {}".format(e)})
        return json.dumps(response)

    @app.route("/conversations/<sender_id>/tracker/events",
               methods=['POST', 'OPTIONS'])
    @check_cors
    @ensure_loaded_agent
    def append_events(self, request, sender_id):
        """Append a list of events to the state of a conversation"""

        request.setHeader('Content-Type', 'application/json')
        request_params = json.loads(request.content.read().decode(
            'utf-8', 'strict'))
        evts = events.deserialise_events(request_params)
        tracker = self.agent.tracker_store.get_or_create_tracker(sender_id)
        for e in evts:
            tracker.update(e)
        self.agent.tracker_store.save(tracker)
        return json.dumps(tracker.current_state())

    @app.route("/conversations/<sender_id>/tracker",
               methods=['GET', 'OPTIONS'])
    @check_cors
    @ensure_loaded_agent
    def retrieve_tracker(self, request, sender_id):
        """Get a dump of a conversations tracker including its events."""

        # parameters
        use_history = bool_arg(request, 'ignore_restarts', default=False)
        should_include_events = bool_arg(request, 'events', default=True)
        until_time = request.args.get('until', None)

        # retrieve tracker and set to requested state
        tracker = self.agent.tracker_store.get_or_create_tracker(sender_id)
        if until_time is not None:
            tracker = tracker.travel_back_in_time(float(until_time))

        # dump and return tracker
        state = tracker.current_state(
            should_include_events=should_include_events,
            only_events_after_latest_restart=use_history)
        return json.dumps(state)

    @app.route("/conversations/<sender_id>/tracker",
               methods=['PUT', 'OPTIONS'])
    @check_cors
    @ensure_loaded_agent
    def update_tracker(self, request, sender_id):
        """Use a list of events to set a conversations tracker to a state."""

        request.setHeader('Content-Type', 'application/json')
        request_params = json.loads(request.content.read().decode(
            'utf-8', 'strict'))
        tracker = DialogueStateTracker.from_dict(sender_id, request_params,
                                                 self.agent.domain)
        self.agent.tracker_store.save(tracker)

        # will override an existing tracker with the same id!
        self.agent.tracker_store.save(tracker)
        return json.dumps(tracker.current_state(should_include_events=True))

    @app.route("/conversations/<sender_id>/parse",
               methods=['GET', 'POST', 'OPTIONS'])
    @check_cors
    @requires_auth
    @ensure_loaded_agent
    def parse(self, request, sender_id):
        request.setHeader('Content-Type', 'application/json')
        if request.method.decode('utf-8', 'strict') == 'GET':
            request_params = {
                key.decode('utf-8', 'strict'):
                value[0].decode('utf-8', 'strict')
                for key, value in request.args.items()
            }
        else:
            content = request.content.read()
            try:
                request_params = json.loads(content.decode('utf-8', 'strict'))
            except ValueError as e:
                logger.error("Failed to decode json during parse request. "
                             "Error: {}. Request content: "
                             "'{}'".format(e, content))
                raise

        if 'query' in request_params:
            message = request_params.pop('query')
        elif 'q' in request_params:
            message = request_params.pop('q')
        else:
            request.setResponseCode(400)
            return json.dumps({"error": "Invalid parse parameter specified"})

        try:
            response = self.agent.start_message_handling(message, sender_id)
            request.setResponseCode(200)
            return json.dumps(response)
        except Exception as e:
            request.setResponseCode(500)
            logger.error("Caught an exception during "
                         "parse: {}".format(e),
                         exc_info=1)
            return json.dumps({"error": "{}".format(e)})

    @app.route("/load", methods=['POST', 'OPTIONS'])
    @check_cors
    def load_model(self, request):
        """Loads a zipped model, replacing the existing one."""

        logger.info("Received new model through REST interface.")
        zipped_path = tempfile.NamedTemporaryFile(delete=False, suffix=".zip")
        zipped_path.close()

        with io.open(zipped_path.name, 'wb') as f:
            f.write(request.args[b'model'][0])
        logger.debug("Downloaded model to {}".format(zipped_path.name))

        zip_ref = zipfile.ZipFile(zipped_path.name, 'r')
        zip_ref.extractall(self.model_directory)
        zip_ref.close()
        logger.debug("Unzipped model to {}".format(
            os.path.abspath(self.model_directory)))

        self.agent = self._create_agent(self.model_directory, self.interpreter,
                                        self.action_factory,
                                        self.tracker_store)
        logger.debug("Finished loading new agent.")
        return json.dumps({'success': 1})

    @app.route("/version", methods=['GET', 'OPTIONS'])
    @check_cors
    def version(self, request):
        """respond with the version number of the installed rasa core."""

        request.setHeader('content-type', 'application/json')
        return json.dumps({'version': __version__})
Beispiel #19
0
import json
from itertools import islice
from ledStrip import ledstrip
from threading import Thread
import Queue
from collections import deque
from time import sleep
from klein import Klein
from twisted.internet import reactor

BRIGHTNESS=50
SHIFTTIME = 5*60

#app = Flask(__name__)
app = Klein()

@app.route("/")
def hello():
    return "Hello World!"

@app.route("/webhook", methods=["POST"])
def webhook(request):
    global states
    try:
        payload = json.loads(request.content.read())
        state = payload.get('details', {}).get('state')

        print 'got state: ', state
        states.append(state.lower())
        states.popleft()
        turnOnTheLights(states, timeoutFlag = True)
Beispiel #20
0
import json
import datetime
from twisted.internet import reactor
from twisted.internet.defer import inlineCallbacks, returnValue
from klein import Klein
from bson import ObjectId
from txmongo import MongoConnection
from klein_babel import gettext, locale_from_request

from umongo import Instance, Document, fields, ValidationError, set_gettext
from umongo.marshmallow_bonus import SchemaFromUmongo


app = Klein()
db = MongoConnection().demo_umongo
instance = Instance(db)
set_gettext(gettext)


class MongoJsonEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, (datetime.datetime, datetime.date)):
            return obj.isoformat()
        elif isinstance(obj, ObjectId):
            return str(obj)
        return json.JSONEncoder.default(self, obj)


def jsonify(request, *args, **kwargs):
    """
    jsonify with support for MongoDB ObjectId
Beispiel #21
0
class ConfigurationAPIUserV1(object):
    """
    A user accessing the API.

    The APIs exposed here typically operate on cluster configuration.  They
    frequently return success results when a configuration change has been made
    durable but has not yet been deployed onto the cluster.
    """
    app = Klein()

    def __init__(self, persistence_service, cluster_state_service):
        """
        :param ConfigurationPersistenceService persistence_service: Service
            for retrieving and setting desired configuration.

        :param ClusterStateService cluster_state_service: Service that
            knows about the current state of the cluster.
        """
        self.persistence_service = persistence_service
        self.cluster_state_service = cluster_state_service

    @app.route("/version", methods=['GET'])
    @user_documentation(
        u"""
        Get the version of Flocker being run.
        """,
        section=u"common",
        header=u"Get Flocker version",
        examples=[u"get version"])
    @structured(
        inputSchema={},
        outputSchema={'$ref': '/v1/endpoints.json#/definitions/versions'},
        schema_store=SCHEMAS
    )
    def version(self):
        """
        Return the ``flocker`` version string.
        """
        return {u"flocker":  __version__}

    @app.route("/configuration/datasets", methods=['GET'])
    @user_documentation(
        u"""
        Get the cluster's dataset configuration.
        """,
        header=u"Get the cluster's dataset configuration",
        examples=[u"get configured datasets"],
        section=u"dataset",
    )
    @structured(
        inputSchema={},
        outputSchema={
            '$ref':
            '/v1/endpoints.json#/definitions/configuration_datasets_list',
        },
        schema_store=SCHEMAS,
    )
    def get_dataset_configuration(self):
        """
        Get the configured datasets.

        :return: A ``list`` of ``dict`` representing each of dataset
            that is configured to exist anywhere on the cluster.
        """
        return list(datasets_from_deployment(self.persistence_service.get()))

    @app.route("/configuration/datasets", methods=['POST'])
    @user_documentation(
        u"""
        Create a new dataset.
        """,
        header=u"Create new dataset",
        examples=[
            u"create dataset",
            u"create dataset with dataset_id",
            u"create dataset with duplicate dataset_id",
            u"create dataset with maximum_size",
            u"create dataset with metadata",
        ],
        section=u"dataset",
    )
    @structured(
        inputSchema={
            '$ref':
            '/v1/endpoints.json#/definitions/configuration_datasets_create'
        },
        outputSchema={
            '$ref':
            '/v1/endpoints.json#/definitions/configuration_datasets'},
        schema_store=SCHEMAS
    )
    def create_dataset_configuration(self, primary, dataset_id=None,
                                     maximum_size=None, metadata=None):
        """
        Create a new dataset in the cluster configuration.

        :param unicode primary: The UUID of the node on which the primary
            manifestation of the dataset will be created.

        :param unicode dataset_id: A unique identifier to assign to the
            dataset.  This is a string giving a UUID (per RFC 4122).  If no
            value is given, one will be generated and returned in the response.
            This is not for easy human use.  For human-friendly identifiers,
            use items in ``metadata``.

        :param maximum_size: Either the maximum number of bytes the dataset
            will be capable of storing or ``None`` to make the dataset size
            unlimited. This may be optional or required depending on the
            dataset backend.

        :param dict metadata: A small collection of unicode key/value pairs to
            associate with the dataset.  These items are not interpreted.  They
            are only stored and made available for later retrieval.  Use this
            for things like human-friendly dataset naming, ownership
            information, etc.

        :return: A ``dict`` describing the dataset which has been added to the
            cluster configuration or giving error information if this is not
            possible.
        """
        if dataset_id is None:
            dataset_id = unicode(uuid4())
        dataset_id = dataset_id.lower()

        if metadata is None:
            metadata = {}

        primary = UUID(hex=primary)

        # Use persistence_service to get a Deployment for the cluster
        # configuration.
        deployment = self.persistence_service.get()
        for node in deployment.nodes:
            for manifestation in node.manifestations.values():
                if manifestation.dataset.dataset_id == dataset_id:
                    raise DATASET_ID_COLLISION

        # XXX Check cluster state to determine if the given primary node
        # actually exists.  If not, raise PRIMARY_NODE_NOT_FOUND.
        # See FLOC-1278

        dataset = Dataset(
            dataset_id=dataset_id,
            maximum_size=maximum_size,
            metadata=pmap(metadata)
        )
        manifestation = Manifestation(dataset=dataset, primary=True)

        primary_node = deployment.get_node(primary)

        new_node_config = primary_node.transform(
            ("manifestations", manifestation.dataset_id), manifestation)
        new_deployment = deployment.update_node(new_node_config)
        saving = self.persistence_service.save(new_deployment)

        def saved(ignored):
            result = api_dataset_from_dataset_and_node(dataset, primary)
            return EndpointResponse(CREATED, result)
        saving.addCallback(saved)
        return saving

    @app.route("/configuration/datasets/<dataset_id>", methods=['DELETE'])
    @user_documentation(
        u"""
        Deletion is idempotent: deleting a dataset multiple times will
        result in the same response.
        """,
        header=u"Delete an existing dataset",
        examples=[
            u"delete dataset",
            u"delete dataset with unknown dataset id",
        ],
        section=u"dataset",
    )
    @structured(
        inputSchema={},
        outputSchema={
            '$ref':
            '/v1/endpoints.json#/definitions/configuration_datasets'},
        schema_store=SCHEMAS
    )
    def delete_dataset(self, dataset_id):
        """
        Delete an existing dataset in the cluster configuration.

       :param unicode dataset_id: The unique identifier of the dataset.  This
            is a string giving a UUID (per RFC 4122).

        :return: A ``dict`` describing the dataset which has been marked
            as deleted in the cluster configuration or giving error
            information if this is not possible.
        """
        # Get the current configuration.
        deployment = self.persistence_service.get()

        # XXX this doesn't handle replicas
        # https://clusterhq.atlassian.net/browse/FLOC-1240
        old_manifestation, origin_node = _find_manifestation_and_node(
            deployment, dataset_id)

        new_node = origin_node.transform(
            ("manifestations", dataset_id, "dataset", "deleted"), True)
        deployment = deployment.update_node(new_node)

        saving = self.persistence_service.save(deployment)

        def saved(ignored):
            result = api_dataset_from_dataset_and_node(
                new_node.manifestations[dataset_id].dataset, new_node.uuid,
            )
            return EndpointResponse(OK, result)
        saving.addCallback(saved)
        return saving

    @app.route("/configuration/datasets/<dataset_id>", methods=['POST'])
    @user_documentation(
        u"""
        This can be used to:

        * Move a dataset from one node to another by changing the
          ``primary`` attribute.
        * In the future update metadata.

        """,
        header=u"Update an existing dataset",
        examples=[
            u"update dataset with primary",
            u"update dataset with unknown dataset id",
        ],
        section=u"dataset",
    )
    @structured(
        inputSchema={
            '$ref':
            '/v1/endpoints.json#/definitions/configuration_datasets_update'},
        outputSchema={
            '$ref':
            '/v1/endpoints.json#/definitions/configuration_datasets'},
        schema_store=SCHEMAS
    )
    def update_dataset(self, dataset_id, primary=None):
        """
        Update an existing dataset in the cluster configuration.

        :param unicode dataset_id: The unique identifier of the dataset.  This
            is a string giving a UUID (per RFC 4122).

        :param primary: The UUID of the node to which the dataset will be
            moved, or ``None`` indicating no change.

        :return: A ``dict`` describing the dataset which has been added to the
            cluster configuration or giving error information if this is not
            possible.
        """
        # Get the current configuration.
        deployment = self.persistence_service.get()

        # Raises DATASET_NOT_FOUND if the ``dataset_id`` is not found.
        primary_manifestation, current_node = _find_manifestation_and_node(
            deployment, dataset_id
        )

        if primary_manifestation.dataset.deleted:
            raise DATASET_DELETED

        if primary is not None:
            deployment = _update_dataset_primary(
                deployment, dataset_id, UUID(hex=primary)
            )

        saving = self.persistence_service.save(deployment)

        primary_manifestation, current_node = _find_manifestation_and_node(
            deployment, dataset_id
        )

        # Return an API response dictionary containing the dataset with updated
        # primary address.
        def saved(ignored):
            result = api_dataset_from_dataset_and_node(
                primary_manifestation.dataset,
                current_node.uuid,
            )
            return EndpointResponse(OK, result)
        saving.addCallback(saved)
        return saving

    @app.route("/state/datasets", methods=['GET'])
    @user_documentation(
        u"""
        The result reflects the control service's knowledge, which may be
        out of date or incomplete. E.g. a dataset agent has not connected
        or updated the control service yet.
        """,
        header=u"Get current cluster datasets",
        examples=[u"get state datasets"],
        section=u"dataset",
    )
    @structured(
        inputSchema={},
        outputSchema={
            '$ref': '/v1/endpoints.json#/definitions/state_datasets_array'
            },
        schema_store=SCHEMAS
    )
    def state_datasets(self):
        """
        Return all primary manifest datasets and all non-manifest datasets in
        the cluster.

        :return: A ``list`` containing all datasets in the cluster.
        """
        # XXX This duplicates code in datasets_from_deployment, but that
        # function is designed to operate on a Deployment rather than a
        # DeploymentState instance and the dataset configuration result
        # includes metadata and deleted flags which should not be part of the
        # dataset state response.
        # Refactor. See FLOC-2207.
        response = []
        deployment_state = self.cluster_state_service.as_deployment()
        get_manifestation_path = self.cluster_state_service.manifestation_path

        for dataset, node in deployment_state.all_datasets():
            response_dataset = dict(
                dataset_id=dataset.dataset_id,
            )

            if node is not None:
                response_dataset[u"primary"] = unicode(node.uuid)
                response_dataset[u"path"] = get_manifestation_path(
                    node.uuid,
                    dataset[u"dataset_id"]
                ).path.decode("utf-8")

            if dataset.maximum_size is not None:
                response_dataset[u"maximum_size"] = dataset.maximum_size

            response.append(response_dataset)
        return response

    @app.route("/configuration/containers", methods=['GET'])
    @user_documentation(
        u"""
        These containers may or may not actually exist on the
        cluster.
        """,
        header=u"Get the cluster's container configuration",
        examples=[u"get configured containers"],
        section=u"container",
    )
    @structured(
        inputSchema={},
        outputSchema={
            '$ref':
            '/v1/endpoints.json#/definitions/configuration_containers_array',
        },
        schema_store=SCHEMAS,
    )
    def get_containers_configuration(self):
        """
        Get the configured containers.

        :return: A ``list`` of ``dict`` representing each of the containers
            that are configured to exist anywhere on the cluster.
        """
        return list(containers_from_deployment(self.persistence_service.get()))

    @app.route("/state/containers", methods=['GET'])
    @user_documentation(
        u"""
        This reflects the control service's knowledge of the cluster,
        which may be out of date or incomplete, e.g. if a container agent
        has not connected or updated the control service yet.
        """,
        header=u"Get the cluster's actual containers",
        examples=[u"get actual containers"],
        section=u"container",
    )
    @structured(
        inputSchema={},
        outputSchema={
            '$ref':
            '/v1/endpoints.json#/definitions/state_containers_array',
        },
        schema_store=SCHEMAS,
    )
    def get_containers_state(self):
        """
        Get the containers present in the cluster.

        :return: A ``list`` of ``dict`` representing each of the containers
            that are configured to exist anywhere on the cluster.
        """
        result = []
        deployment_state = self.cluster_state_service.as_deployment()
        for node in deployment_state.nodes:
            if node.applications is None:
                continue
            for application in node.applications:
                container = container_configuration_response(
                    application, node.uuid)
                container[u"running"] = application.running
                result.append(container)
        return result

    def _get_attached_volume(self, node_uuid, volume):
        """
        Create an ``AttachedVolume`` given a volume dictionary.

        :param UUID node_uuid: The node where the volume should be.
        :param dict volume: Parameters for specific volume passed to creation
            endpoint.

        :return AttachedVolume: Corresponding instance.
        """
        deployment = self.persistence_service.get()

        instances = list(manifestations_from_deployment(
            deployment, volume[u"dataset_id"]))

        if not any(m for (m, _) in instances if not m.dataset.deleted):
            raise DATASET_NOT_FOUND
        if not any(n for (_, n) in instances if n.uuid == node_uuid):
            raise DATASET_ON_DIFFERENT_NODE
        if any(app for app in deployment.applications() if
                app.volume and
                app.volume.manifestation.dataset_id == volume[u"dataset_id"]):
            raise DATASET_IN_USE

        return AttachedVolume(
            manifestation=[m for (m, node) in instances
                           if node.uuid == node_uuid and m.primary][0],
            mountpoint=FilePath(volume[u"mountpoint"].encode("utf-8")))

    @app.route("/configuration/containers", methods=['POST'])
    @user_documentation(
        u"""
        The container will be automatically started once it is created on
        the cluster.
        """,
        header=u"Add a new container to the configuration",
        examples=[
            u"create container",
            u"create container with duplicate name",
            u"create container with ports",
            u"create container with environment",
            u"create container with attached volume",
            u"create container with cpu shares",
            u"create container with memory limit",
            u"create container with links",
            u"create container with command line",
            # No example of creating a container with a different restart
            # policy because only the "never" policy is supported.  See
            # FLOC-2449.
        ],
        section=u"container",
    )
    @structured(
        inputSchema={
            '$ref': '/v1/endpoints.json#/definitions/configuration_container'},
        outputSchema={
            '$ref': '/v1/endpoints.json#/definitions/configuration_container'},
        schema_store=SCHEMAS
    )
    def create_container_configuration(
        self, node_uuid, name, image, ports=(), environment=None,
        restart_policy=None, cpu_shares=None, memory_limit=None,
        links=(), volumes=(), command_line=None,
    ):
        """
        Create a new dataset in the cluster configuration.

        :param unicode node_uuid: The UUID of the node on which the container
            will run.

        :param unicode name: A unique identifier for the container within
            the Flocker cluster.

        :param unicode image: The name of the Docker image to use for the
            container.

        :param list ports: A ``list`` of ``dict`` objects, mapping internal
            to external ports for the container.

        :param dict environment: A ``dict`` of key/value pairs to be supplied
            to the container as environment variables. Keys and values must be
            ``unicode``.

        :param dict restart_policy: A restart policy for the container, this
            is a ``dict`` with at a minimum a "name" key, whose value must be
            one of "always", "never" or "on-failure". If the "name" is given
            as "on-failure", there may also be another optional key
            "maximum_retry_count", containing a positive ``int`` specifying
            the maximum number of times we should attempt to restart a failed
            container.

        :param volumes: A iterable of ``dict`` with ``"dataset_id"`` and
            ``"mountpoint"`` keys.

        :param int cpu_shares: A positive integer specifying the relative
            weighting of CPU cycles for this container (see Docker's run
            reference for further information).

        :param int memory_limit: A positive integer specifying the maximum
            amount of memory in bytes available to this container.

        :param list links: A ``list`` of ``dict`` objects, mapping container
            links via "alias", "local_port" and "remote_port" values.

        :param command_line: If not ``None``, the command line to use when
            running the Docker image's entry point.

        :return: An ``EndpointResponse`` describing the container which has
            been added to the cluster configuration.
        """
        deployment = self.persistence_service.get()

        node_uuid = UUID(hex=node_uuid)

        # Check if container by this name already exists, if it does
        # return error.
        for node in deployment.nodes:
            for application in node.applications:
                if application.name == name:
                    raise CONTAINER_NAME_COLLISION

        # Find the volume, if any; currently we only support one volume
        # https://clusterhq.atlassian.net/browse/FLOC-49
        attached_volume = None
        if volumes:
            attached_volume = self._get_attached_volume(node_uuid, volumes[0])

        # Find the node.
        node = deployment.get_node(node_uuid)

        # Check if we have any ports in the request. If we do, check existing
        # external ports exposed to ensure there is no conflict. If there is a
        # conflict, return an error.
        for port in ports:
            for current_node in deployment.nodes:
                for application in current_node.applications:
                    for application_port in application.ports:
                        if application_port.external_port == port['external']:
                            raise CONTAINER_PORT_COLLISION

        # If links are present, check that there are no conflicts in local
        # ports or alias names.
        link_aliases = set()
        link_local_ports = set()
        application_links = set()
        for link in links:
            if link['alias'] in link_aliases:
                raise LINK_ALIAS_COLLISION
            if link['local_port'] in link_local_ports:
                raise LINK_PORT_COLLISION
            link_aliases.add(link['alias'])
            link_local_ports.add(link['local_port'])
            application_links.add(
                Link(
                    alias=link['alias'], local_port=link['local_port'],
                    remote_port=link['remote_port']
                )
            )

        # If we have ports specified, add these to the Application instance.
        application_ports = []
        for port in ports:
            application_ports.append(Port(
                internal_port=port['internal'],
                external_port=port['external']
            ))

        if environment is not None:
            environment = frozenset(environment.items())

        if restart_policy is None:
            restart_policy = dict(name=u"never")

        policy_name = restart_policy.pop("name")
        policy_factory = FLOCKER_RESTART_POLICY_NAME_TO_POLICY[policy_name]
        policy = policy_factory(**restart_policy)

        # Create Application object, add to Deployment, save.
        application = Application(
            name=name,
            image=DockerImage.from_string(image),
            ports=frozenset(application_ports),
            environment=environment,
            volume=attached_volume,
            restart_policy=policy,
            cpu_shares=cpu_shares,
            memory_limit=memory_limit,
            links=application_links,
            command_line=command_line,
        )

        new_node_config = node.transform(
            ["applications"],
            lambda s: s.add(application)
        )

        new_deployment = deployment.update_node(new_node_config)
        saving = self.persistence_service.save(new_deployment)

        # Return passed in dictionary with CREATED response code.
        def saved(_):
            result = container_configuration_response(application, node_uuid)
            return EndpointResponse(CREATED, result)
        saving.addCallback(saved)
        return saving

    @app.route("/configuration/containers/<name>", methods=['POST'])
    @user_documentation(
        u"""
        This will lead to the container being relocated to the specified host
        and restarted. This will also update the primary host of any attached
        datasets.
        """,
        header=u"Update a named container's configuration",
        examples=[u"move container"],
        section=u"container",
    )
    @structured(
        inputSchema={
            '$ref':
            '/v1/endpoints.json#/definitions/configuration_container_update',
        },
        outputSchema={
            '$ref':
            '/v1/endpoints.json#/definitions/configuration_container',
        },
        schema_store=SCHEMAS,
    )
    def update_containers_configuration(self, name, node_uuid):
        """
        Update the specified container's configuration.

        :param unicode name: A unique identifier for the container within
            the Flocker cluster.

        :param unicode node_uuid: The address of the node on which the
            container will run.

        :return: An ``EndpointResponse`` describing the container which has
            been updated.
        """
        deployment = self.persistence_service.get()
        node_uuid = UUID(hex=node_uuid)
        target_node = deployment.get_node(node_uuid)
        for node in deployment.nodes:
            for application in node.applications:
                if application.name == name:
                    deployment = deployment.move_application(
                        application, target_node
                    )
                    saving = self.persistence_service.save(deployment)

                    def saved(_):
                        result = container_configuration_response(
                            application, node_uuid
                        )
                        return EndpointResponse(OK, result)

                    saving.addCallback(saved)
                    return saving

        # Didn't find the application:
        raise CONTAINER_NOT_FOUND

    @app.route("/configuration/containers/<name>", methods=['DELETE'])
    @user_documentation(
        u"""
        This will lead to the container being stopped and not being
        restarted again. Any datasets that were attached as volumes will
        continue to exist on the cluster.
        """,
        header=u"Remove a container from the configuration",
        examples=[
            u"remove a container",
            u"remove a container with unknown name",
        ],
        section=u"container",
    )
    @structured(
        inputSchema={},
        outputSchema={},
        schema_store=SCHEMAS
    )
    def delete_container_configuration(self, name):
        """
        Remove a container from the cluster configuration.

        :param unicode name: A unique identifier for the container within
            the Flocker cluster.

        :return: An ``EndpointResponse``.
        """
        deployment = self.persistence_service.get()

        for node in deployment.nodes:
            for application in node.applications:
                if application.name == name:
                    updated_node = node.transform(
                        ["applications"], lambda s: s.remove(application))
                    d = self.persistence_service.save(
                        deployment.update_node(updated_node))
                    d.addCallback(lambda _: None)
                    return d

        # Didn't find the application:
        raise CONTAINER_NOT_FOUND

    @app.route("/state/nodes", methods=['GET'])
    @user_documentation(
        u"""
        Some nodes may not be listed if their agents are disconnected from
        the cluster. IP addresses may be private IP addresses that are not
        publicly routable.
        """,
        header=u"List known nodes in the cluster",
        examples=[
            u"list known nodes",
        ],
        section=u"common",
    )
    @structured(
        inputSchema={},
        outputSchema={"$ref":
                      '/v1/endpoints.json#/definitions/nodes_array'},
        schema_store=SCHEMAS
    )
    def list_current_nodes(self):
        return [{u"host": node.hostname, u"uuid": unicode(node.uuid)}
                for node in
                self.cluster_state_service.as_deployment().nodes]

    @app.route("/configuration/_compose", methods=['POST'])
    @private_api
    @structured(
        inputSchema={
            '$ref':
            '/v1/endpoints.json#/definitions/configuration_compose'
        },
        outputSchema={},
        schema_store=SCHEMAS
    )
    def replace_configuration(self, applications, deployment):
        """
        Replace the existing configuration with one given by flocker-deploy
        command line tool.

        :param applications: Configuration in Flocker-native or
            Fig/Compose format.

        :param deployment: Configuration of which applications run on
            which nodes.
        """
        try:
            configuration = FigConfiguration(applications)
            if not configuration.is_valid_format():
                configuration = FlockerConfiguration(applications)
            return self.persistence_service.save(model_from_configuration(
                deployment_state=self.cluster_state_service.as_deployment(),
                applications=configuration.applications(),
                deployment_configuration=deployment))
        except ConfigurationError as e:
            raise make_bad_request(code=BAD_REQUEST, description=unicode(e))
Beispiel #22
0
class RasaNLU(object):
    """Class representing Rasa NLU http server"""

    app = Klein()

    def __init__(self,
                 data_router,
                 loglevel='INFO',
                 logfile=None,
                 num_threads=1,
                 token=None,
                 cors_origins=None,
                 testing=False,
                 default_config_path=None):

        self._configure_logging(loglevel, logfile)

        self.default_model_config = self._load_default_config(
            default_config_path)

        self.data_router = data_router
        self._testing = testing
        self.cors_origins = cors_origins if cors_origins else ["*"]
        self.access_token = token
        reactor.suggestThreadPoolSize(num_threads * 5)

    @staticmethod
    def _load_default_config(path):
        if path:
            return config.load(path).as_dict()
        else:
            return {}

    @staticmethod
    def _configure_logging(loglevel, logfile):
        logging.basicConfig(filename=logfile, level=loglevel)
        logging.captureWarnings(True)

    @app.route("/", methods=['GET', 'OPTIONS'])
    @check_cors
    def hello(self, request):
        """Main Rasa route to check if the server is online"""
        return "hello from Rasa NLU: " + __version__

    @app.route("/parse", methods=['GET', 'POST', 'OPTIONS'])
    @requires_auth
    @check_cors
    @inlineCallbacks
    def parse(self, request):
        request.setHeader('Content-Type', 'application/json')
        if request.method.decode('utf-8', 'strict') == 'GET':
            request_params = decode_parameters(request)
        else:
            request_params = simplejson.loads(request.content.read().decode(
                'utf-8', 'strict'))

        if 'query' in request_params:
            request_params['q'] = request_params.pop('query')

        if 'q' not in request_params:
            request.setResponseCode(404)
            dumped = json_to_string(
                {"error": "Invalid parse parameter specified"})
            returnValue(dumped)
        else:
            data = self.data_router.extract(request_params)
            try:
                request.setResponseCode(200)
                response = yield (self.data_router.parse(data)
                                  if self._testing else threads.deferToThread(
                                      self.data_router.parse, data))
                returnValue(json_to_string(response))
            except InvalidProjectError as e:
                request.setResponseCode(404)
                returnValue(json_to_string({"error": "{}".format(e)}))
            except Exception as e:
                request.setResponseCode(500)
                logger.exception(e)
                returnValue(json_to_string({"error": "{}".format(e)}))

    @app.route("/version", methods=['GET', 'OPTIONS'])
    @requires_auth
    @check_cors
    def version(self, request):
        """Returns the Rasa server's version"""

        request.setHeader('Content-Type', 'application/json')
        return json_to_string({'version': __version__})

    @app.route("/config", methods=['GET', 'OPTIONS'])
    @requires_auth
    @check_cors
    def rasaconfig(self, request):
        """Returns the in-memory configuration of the Rasa server"""

        # DEPRECATED: I don't think there is a use case for this endpoint
        # anymore - when training a new model, the user should always post
        # the configuration as part of the request instead of relying on
        # the servers config.
        request.setHeader('Content-Type', 'application/json')
        return json_to_string(self.default_model_config)

    @app.route("/status", methods=['GET', 'OPTIONS'])
    @requires_auth
    @check_cors
    def status(self, request):
        request.setHeader('Content-Type', 'application/json')
        return json_to_string(self.data_router.get_status())

    @app.route("/train", methods=['POST', 'OPTIONS'])
    @requires_auth
    @check_cors
    @inlineCallbacks
    def train(self, request):
        # if not set will use the default project name, e.g. "default"
        project = parameter_or_default(request, "project", default=None)
        # if set will not generate a model name but use the passed one
        model_name = parameter_or_default(request, "model", default=None)

        request_content = request.content.read().decode('utf-8', 'strict')

        if is_yaml_request(request):
            # assumes the user submitted a model configuration with a data
            # parameter attached to it
            model_config = utils.read_yaml(request_content)
            data = model_config.get("data")
        else:
            # assumes the caller just provided training data without config
            # this will use the default model config the server
            # was started with
            model_config = self.default_model_config
            data = request_content

        data_file = dump_to_data_file(data)

        request.setHeader('Content-Type', 'application/json')

        try:
            request.setResponseCode(200)

            response = yield self.data_router.start_train_process(
                data_file, project, RasaNLUModelConfig(model_config),
                model_name)

            returnValue(
                json_to_string(
                    {'info': 'new model trained: {}'
                     ''.format(response)}))
        except AlreadyTrainingError as e:
            request.setResponseCode(403)
            returnValue(json_to_string({"error": "{}".format(e)}))
        except InvalidProjectError as e:
            request.setResponseCode(404)
            returnValue(json_to_string({"error": "{}".format(e)}))
        except TrainingException as e:
            request.setResponseCode(500)
            returnValue(json_to_string({"error": "{}".format(e)}))

    @app.route("/evaluate", methods=['POST', 'OPTIONS'])
    @requires_auth
    @check_cors
    @inlineCallbacks
    def evaluate(self, request):
        data_string = request.content.read().decode('utf-8', 'strict')
        params = {
            key.decode('utf-8', 'strict'): value[0].decode('utf-8', 'strict')
            for key, value in request.args.items()
        }

        request.setHeader('Content-Type', 'application/json')

        try:
            request.setResponseCode(200)
            response = yield self.data_router.evaluate(data_string,
                                                       params.get('project'),
                                                       params.get('model'))
            returnValue(json_to_string(response))
        except Exception as e:
            request.setResponseCode(500)
            returnValue(json_to_string({"error": "{}".format(e)}))

    @app.route("/models", methods=['DELETE', 'OPTIONS'])
    @requires_auth
    @check_cors
    def unload_model(self, request):
        params = {
            key.decode('utf-8', 'strict'): value[0].decode('utf-8', 'strict')
            for key, value in request.args.items()
        }

        request.setHeader('Content-Type', 'application/json')
        try:
            request.setResponseCode(200)
            response = self.data_router.unload_model(
                params.get('project', RasaNLUModelConfig.DEFAULT_PROJECT_NAME),
                params.get('model'))
            return simplejson.dumps(response)
        except Exception as e:
            request.setResponseCode(500)
            logger.exception(e)
            return simplejson.dumps({"error": "{}".format(e)})
Beispiel #23
0
    def test_example(self):
        """
        When the endpoint being documented references an example HTTP session
        that session is included in the generated API documentation, marked up
        as HTTP.
        """
        app = Klein()

        @app.route("/", methods=["GET"])
        @user_documentation("""
            Demonstrates examples.
            """, ["example-example"])
        def hasExamples():
            pass

        examples = {
            u"example-example": {
                u"id": u"example-example",
                u"doc": u"This example demonstrates examples.",
                u"request": u"GET /prefix/ HTTP/1.1\n"
                u"\n",
                u"response": u"HTTP/1.1 200 OK\n"
                u"\n"
                u'"%(DOMAIN)s"\n',
            },
        }

        rest = list(makeRst(b"/prefix", app, examples.get, {}))
        self.assertEqual(
            [
                '',
                # This line introduces the endpoint
                '.. http:get:: /prefix/',
                '',
                # Here is the prose documentation for the endpoint.
                '   Demonstrates examples.',
                '   ',
                '   **Example:** This example demonstrates examples.',
                '   ',
                # This is a header introducing the request portion of the session.
                '   Request',
                # This blank line is necessary to satisfy reST for some reason.
                '   ',
                '   .. sourcecode:: http',
                '   ',
                # Here's the bytes of the HTTP request.
                '      GET /prefix/ HTTP/1.1',
                '      Host: api.example.com',
                '      Content-Type: application/json',
                '      ',
                # This blank line is necessary to satisfy reST for some reason.
                '   ',
                # The same again but for the HTTP response.
                '   Response',
                '   ',
                '   .. sourcecode:: http',
                '   ',
                '      HTTP/1.1 200 OK',
                '      Content-Type: application/json',
                '      ',
                '      "example.com"',
                '   ',
                '',
            ],
            rest)
class Server:
    app = Klein()

    def __init__(self, model_directory, interpreter):
        self.model_directory = model_directory
        self.interpreter = interpreter
        self.agent = self._create_agent(model_directory, interpreter)

    @staticmethod
    def _create_agent(model_directory, interpreter):
        """Creates a Rasa Agent which runs when the server is started"""
        try:
            endpoints = AvailableEndpoints.read_endpoints('endpoints.yml')
            return Agent.load(model_directory, interpreter, action_endpoint=endpoints.action)
        except Exception as e:
            logger.warn("Failed to load any agent model. Running "
                        "Rasa Core server with out loaded model now. {}"
                        "".format(e))
            return None

    @app.route("/api/v1/status", methods=['GET'])
    def status(self, request):
        """Check if the server is running and responds with the status."""
        request.setHeader('Access-Control-Allow-Origin', '*')
        return json.dumps({'status': 'OK'})

    @app.route('/api/v1/<sender_id>/parse', methods=['GET', 'POST'])
    def parse(self, request, sender_id):
        request.setHeader('Content-Type', 'application/json')
        request_params = request_parameters(request)

        if 'query' in request_params:
            message = request_params.pop('query')
        elif 'q' in request_params:
            message = request_params.pop('q')
        else:
            request.setResponseCode(400)
            return json.dumps({"error": "Invalid parse parameter specified"})
        try:
            response = self.agent.handle_message(message, sender_id)
            request.setResponseCode(200)
            return json.dumps(response)
        except Exception as e:
            request.setResponseCode(500)
            logger.error("Caught an exception during "
                         "parse: {}".format(e), exc_info=1)
            return json.dumps({"error": "{}".format(e)})

    @app.route('/api/v1/<sender_id>/respond', methods=['GET', 'POST'])
    def respond(self, request, sender_id):
        request.setHeader('Content-Type', 'application/json')
        request.setHeader('Access-Control-Allow-Origin', '*')
        request_params = request_parameters(request)
        if 'query' in request_params:
            message = request_params.pop('query')
        elif 'q' in request_params:
            message = request_params.pop('q')
        else:
            request.setResponseCode(400)
            return json.dumps({"error": "Invalid parse parameter specified"})
        try:
            out = CollectingOutputChannel()
            response = self.agent.handle_text(message, output_channel=out, sender_id=sender_id)
            request.setResponseCode(200)
            return json.dumps(response)
        except Exception as e:
            request.setResponseCode(500)
            logger.error("Caught an exception during "
                         "parse: {}".format(e), exc_info=1)
            return json.dumps({"error": "{}".format(e)})
Beispiel #25
0
    Y_pred_text = text_pred

    # construct output, highlight diacritics
    rez = []
    for ch0, ch2 in zip(text0, Y_pred_text):
        ch = ch0
        if ch0.lower() != ch2.lower() and ch2 != " ":
            ch = "<span class='mod'>" + ch2 + "</span>"
        rez.append(ch)
    rez_str = "".join(rez).strip()
    return rez_str


#DBG()

app = Klein()


@app.route("/ajax")
def generate_ajax(request):
    txt = request.content.read().decode("utf-8")
    print("GOT TXT=", txt, type(txt))
    request.setHeader('Content-Type', 'text/html; charset=utf-8')
    request.write(predict(model, txt).encode("utf-8"))


@app.route("/", branch=True)
def generate_index(request):
    return File("./app")

Beispiel #26
0
class RestApi:
    app = Klein()
    notif = None

    def __init__(self):
        self.notif = NotificationDB.instance()

    #
    # REST API Routes
    #
    @app.route('/')
    def home(self, request):
        endpoints_html = """<ul>
            <li><pre>{apiPrefix}/notifications/block/&lt;height&gt;</pre> <em>notifications by block</em></li>
            <li><pre>{apiPrefix}/notifications/addr/&lt;addr&gt;</pre><em>notifications by address</em></li>
            <li><pre>{apiPrefix}/notifications/tx/&lt;hash&gt;</pre><em>notifications by tx</em></li>
            <li><pre>{apiPrefix}/notifications/contract/&lt;hash&gt;</pre><em>notifications by contract</em></li>
            <li><pre>{apiPrefix}/tokens</pre><em>lists all NEP5 Tokens</em></li>
            <li><pre>{apiPrefix}/token/&lt;contract_hash&gt;</pre><em>list an NEP5 Token</em></li>
            <li><pre>{apiPrefix}/status</pre> <em>current block height and version</em></li>
        </ul>
        """.format(apiPrefix=API_URL_PREFIX)

        return """<html>
                    <style>body {padding:20px;max-width:800px;pre { background-color:#eee; }</style>
                    <body>
                        <p>
                            <h2>REST API for NEO %s</h2>
                            (see also <a href="https://github.com/CityOfZion/neo-python">neo-python</a>, <a href="https://github.com/CityOfZion/neo-python/blob/development/api-server.py">api-server.py</a>)
                        </p>

                        <hr/>

                        <h2>endpoints:</h2>
                        <p>%s</p>
                        <div>
                            <hr/>
                            <h3>pagination</h3>
                            <p>results are offered in page size of 500</p>
                            <p>you may request a different page by specifying the <code>page</code> query string param, for example:</p>
                            <pre>/block/123456?page=3</pre>
                            <p>page index starts at 0, so the 2nd page would be <code>?page=1</code></p>
                            <hr/>
                            <h3>sample output</h3>
                            <pre>
{
    "page_len": 1000,
    "total_pages": 1,
    "total": 4,
    "page": 1,
    "current_height": 982506,
    "results": [
        {
            "type": "SmartContract.Runtime.Notify",
            "contract": "400cbed5b41014788d939eaf6286e336e7140f8c",
            "tx": "d0805fd7ec19a4a414374ae3720447d2576659053eb7588b85a0f9f1fd629791",
            "block": 928119,
            "addr_from": "AUYSKFEWPZxP57fo3TsK6Lwg22qxSFupKF",
            "amount": 1,
            "addr_to": "ALULT5WpeiHnEXYFe72Yq7nRB3ZBmsBypq",
            "notify_type": "transfer"
        },
        {
            "type": "SmartContract.Runtime.Notify",
            "contract": "d3de84c166d93ad2581cb587bda8e02b12dc37ca",
            "tx": "667df082eaa16ce2b07e48e214eb019b3e9450e76daea4f5b0450578a07836ef",
            "block": 936352,
            "addr_from": "ALULT5WpeiHnEXYFe72Yq7nRB3ZBmsBypq",
            "amount": 1,
            "addr_to": "AaD74SkQXsSE7UtSutQ4VV3mRdQUoMk98X",
            "notify_type": "transfer"
        },
        {
            "type": "SmartContract.Runtime.Notify",
            "contract": "2c0fdfa9592814b0a938219e218e3a6b08615acd",
            "tx": "eda792e7814e128eecda992f78a11577ee0604827de4aa91ffcda4616c889191",
            "block": 939449,
            "addr_from": "ALULT5WpeiHnEXYFe72Yq7nRB3ZBmsBypq",
            "amount": 1,
            "addr_to": "AaVgSU9vEPdwc49rPrCyj1LYkpsGFNgbjy",
            "notify_type": "transfer"
        },
        {
            "type": "SmartContract.Runtime.Notify",
            "contract": "f9572c5b119a6b5775a6af07f1cef5d310038f55",
            "tx": "6d0f1decbf3874d08d41f2cc9e8672cd3507c962668c15793e3dd3e01fc3551c",
            "block": 942369,
            "addr_from": "ALULT5WpeiHnEXYFe72Yq7nRB3ZBmsBypq",
            "amount": 1,
            "addr_to": "APaGQT4dx4gUDApVPnbtZvChJ8UKRsZBdt",
            "notify_type": "transfer"
        }
    ],
    "message": ""
}</pre>
                        </div>
                    </body>
                </html>""" % (settings.net_name, endpoints_html)

    @app.route('%s/notifications/block/<int:block>' % API_URL_PREFIX,
               methods=['GET'])
    @cors_header
    def get_by_block(self, request, block):
        request.setHeader('Content-Type', 'application/json')
        try:
            if int(block) > Blockchain.Default().Height:
                return self.format_message("Higher than current block")
            else:
                notifications = self.notif.get_by_block(block)
        except Exception as e:
            logger.info("Could not get notifications for block %s %s" %
                        (block, e))
            return self.format_message(
                "Could not get notifications for block %s because %s " %
                (block, e))
        return self.format_notifications(request, notifications)

    @app.route('%s/notifications/addr/<string:address>' % API_URL_PREFIX,
               methods=['GET'])
    @cors_header
    def get_by_addr(self, request, address):
        request.setHeader('Content-Type', 'application/json')
        try:
            notifications = self.notif.get_by_addr(address)
        except Exception as e:
            logger.info("Could not get notifications for address %s " %
                        address)
            return self.format_message(
                "Could not get notifications for address %s because %s" %
                (address, e))
        return self.format_notifications(request, notifications)

    @app.route('%s/notifications/tx/<string:tx_hash>' % API_URL_PREFIX,
               methods=['GET'])
    @cors_header
    def get_by_tx(self, request, tx_hash):
        request.setHeader('Content-Type', 'application/json')

        bc = Blockchain.Default()  # type: Blockchain
        notifications = []
        try:
            hash = UInt256.ParseString(tx_hash)
            tx, height = bc.GetTransaction(hash)
            if not tx:
                return self.format_message(
                    "Could not find transaction for hash %s" % (tx_hash))
            block_notifications = self.notif.get_by_block(height - 1)
            for n in block_notifications:
                if n.tx_hash == tx.Hash:
                    notifications.append(n)
        except Exception as e:
            logger.info("Could not get tx with hash %s because %s " %
                        (tx_hash, e))
            return self.format_message(
                "Could not get tx with hash %s because %s " % (tx_hash, e))

        return self.format_notifications(request, notifications)

    @app.route('%s/notifications/contract/<string:contract_hash>' %
               API_URL_PREFIX,
               methods=['GET'])
    @cors_header
    def get_by_contract(self, request, contract_hash):
        request.setHeader('Content-Type', 'application/json')
        try:
            hash = UInt160.ParseString(contract_hash)
            notifications = self.notif.get_by_contract(hash)
        except Exception as e:
            logger.info("Could not get notifications for contract %s " %
                        contract_hash)
            return self.format_message(
                "Could not get notifications for contract hash %s because %s" %
                (contract_hash, e))
        return self.format_notifications(request, notifications)

    @app.route('%s/tokens' % API_URL_PREFIX, methods=['GET'])
    @cors_header
    def get_tokens(self, request):
        request.setHeader('Content-Type', 'application/json')
        notifications = self.notif.get_tokens()
        return self.format_notifications(request, notifications)

    @app.route('%s/token/<string:contract_hash>' % API_URL_PREFIX,
               methods=['GET'])
    @cors_header
    def get_token(self, request, contract_hash):
        request.setHeader('Content-Type', 'application/json')
        try:
            uint160 = UInt160.ParseString(contract_hash)
            contract_event = self.notif.get_token(uint160)
            if not contract_event:
                return self.format_message(
                    "Could not find contract with hash %s" % contract_hash)
            notifications = [contract_event]
        except Exception as e:
            logger.info("Could not get contract with hash %s because %s " %
                        (contract_hash, e))
            return self.format_message(
                "Could not get contract with hash %s because %s " %
                (contract_hash, e))

        return self.format_notifications(request, notifications)

    @app.route('%s/status' % API_URL_PREFIX, methods=['GET'])
    @cors_header
    def get_status(self, request):
        request.setHeader('Content-Type', 'application/json')
        return json.dumps(
            {
                'current_height': Blockchain.Default().Height + 1,
                'version': settings.VERSION_NAME,
                'num_peers': len(NodeLeader.Instance().Peers)
            },
            indent=4,
            sort_keys=True)

    def format_notifications(self, request, notifications, show_none=False):

        notif_len = len(notifications)
        page_len = 500
        page = 1
        message = ''
        if b'page' in request.args:
            try:
                page = int(request.args[b'page'][0])
            except Exception as e:
                print("could not get page: %s" % e)
        if b'pagesize' in request.args:
            try:
                page_len = int(request.args[b'pagesize'][0])
            except Exception as e:
                print("could not get page length: %s" % e)

        # note, we want pages to start from 1, not 0, to be
        # in sync with C# version
        # we'll also convert page 0 to page1
        if page == 0:
            page = 1

        start = page_len * (page - 1)
        end = start + page_len

        if start > notif_len:
            message = 'page greater than result length'

        notifications = notifications[start:end]
        total_pages = math.ceil(notif_len / page_len)

        return json.dumps(
            {
                'current_height':
                Blockchain.Default().Height + 1,
                'message':
                message,
                'total':
                notif_len,
                'results':
                None if show_none else [n.ToJson() for n in notifications],
                'page':
                page,
                'page_len':
                page_len,
                'total_pages':
                total_pages
            },
            indent=4,
            sort_keys=True)

    def format_message(self, message):
        return json.dumps(
            {
                'current_height': Blockchain.Default().Height + 1,
                'message': message,
                'total': 0,
                'results': None,
                'page': 0,
                'page_len': 0,
                'total_pages': 0
            },
            indent=4,
            sort_keys=True)
Beispiel #27
0
from klein import Klein
app = Klein()


@app.route('/user/<username>')
def pg_user(request, username):
    return 'Hi %s!' % (username, )


app.run("localhost", 8080)
Beispiel #28
0
class MarathonAcmeServer(object):

    app = Klein()
    log = Logger()

    def __init__(self, responder_resource):
        """
        :param responder_resource:
            An ``IResponse`` used to respond to ACME HTTP challenge validation
            requests.
        """
        self.responder_resource = responder_resource
        self.health_handler = None

    def listen(self, reactor, endpoint_description):
        """
        Run the server, i.e. start listening for requests on the given host and
        port.

        :param reactor: The ``IReactorTCP`` to use.
        :param endpoint_description:
            The Twisted description for the endpoint to listen on.
        :return:
            A deferred that returns an object that provides ``IListeningPort``.
        """
        endpoint = serverFromString(reactor, endpoint_description)
        return endpoint.listen(Site(self.app.resource()))

    @app.route('/.well-known/acme-challenge/', branch=True, methods=['GET'])
    def acme_challenge(self, request):
        """
        Respond to ACME challenge validation requests on
        ``/.well-known/acme-challenge/`` using the ACME responder resource.
        """
        return self.responder_resource

    def set_health_handler(self, health_handler):
        """
        Set the handler for the health endpoint.

        :param health_handler:
            The handler for health status requests. This must be a callable
            that returns a Health object.
        """
        self.health_handler = health_handler

    @app.route('/health', methods=['GET'])
    def health(self, request):
        """ Listens to incoming health checks from Marathon on ``/health``. """
        if self.health_handler is None:
            return self._no_health_handler(request)

        health = self.health_handler()
        response_code = OK if health.healthy else SERVICE_UNAVAILABLE
        request.setResponseCode(response_code)
        write_request_json(request, health.json_message)

    def _no_health_handler(self, request):
        self.log.warn('Request to /health made but no handler is set')
        request.setResponseCode(NOT_IMPLEMENTED)
        write_request_json(
            request,
            {'error': 'Cannot determine service health: no handler set'})
Beispiel #29
0
if sys.version_info[0] == 3:
    xrange = range

DBDRIVER = 'mysql'
DBHOSTNAME = os.environ.get('DBHOST', 'localhost')
DATABASE_URI = '%s://benchmarkdbuser:benchmarkdbpass@%s:3306/hello_world?charset=utf8' % (DBDRIVER, DBHOSTNAME)

Base = declarative_base()
db_engine = create_engine(DATABASE_URI)
Session = sessionmaker(bind=db_engine)
db_session = Session()

env = Environment(loader=PackageLoader("app", "templates"), autoescape=True, auto_reload=False)

app = Klein()


class Fortune(Base):
    __tablename__ = "Fortune"
    id = Column(Integer, primary_key=True)
    message = Column(String)

    def serialize(self):
        return {
            'id': self.id,
            'randomNumber': self.randomNumber,
        }

class World(Base):
    __tablename__ = "World"
Beispiel #30
0
    def test_inputSchema(self):
        """
        The generated API documentation includes the input schema.
        """
        app = Klein()

        @app.route(b"/", methods=[b"GET"])
        @structured(
            inputSchema={'$ref': '/v0/test.json#/endpoint'},
            outputSchema={},
            schema_store=self.INPUT_SCHEMAS,
        )
        def f():
            """
            Developer docs,
            """

        rest = list(makeRst(b"/prefix", app, None, self.INPUT_SCHEMAS))

        self.assertEqual(
            rest,
            [
                '',
                '.. http:get:: /prefix/',
                '',
                '   Undocumented.',
                '   ',
                '   .. hidden-code-block:: json',
                '       :label: + Request JSON Schema',
                '       :starthidden: True',
                '   ',
                '       {',
                '           "$schema": "http://json-schema.org/draft-04/schema#",',
                '           "properties": {',
                '               "optional": {',
                '                   "description": "one\\ntwo",',
                '                   "title": "TITLE",',
                '                   "type": "string"',
                '               },',
                '               "param": {',
                '                   "description": "one\\ntwo",',
                '                   "title": "TITLE",',
                '                   "type": "string"',
                '               }',
                '           },',
                '           "required": [',
                '               "param"',
                '           ],',
                '           "type": "object"',
                '       }',
                '   ',
                # YAML is unorderd :(
                '   :<json string optional: TITLE',
                '   ',
                '      one',
                '      two',
                '      ',
                '   :<json string param: *(required)* TITLE',
                '   ',
                '      one',
                '      two',
                '      ',
                '',
            ])
                api,
                "execute",
                capability_version,
                capability_hash, '')

            links.append({"href": capability_url, "rel": "capability"})

        return links
    else:
        return url


def transaction_id(request):
    """
    Extract the transaction id from the given request.

    :param IRequest request: The request we are trying to get the
        transaction id for.

    :returns: A string transaction id.
    """
    return request.responseHeaders.getRawHeaders('X-Response-Id')[0]


app = Klein()
app.route = partial(app.route, strict_slashes=False)

root = Resource()
root.putChild('v1.0', app.resource())
root.putChild('', Data('', 'text/plain'))
Beispiel #32
0
class ConfigApi(object):
    """
    Configuration API.

    This class handles HTTP API calls related to router configuration.
    """

    routes = Klein()

    def __init__(self, update_manager, update_fetcher):
        self.update_manager = update_manager
        self.update_fetcher = update_fetcher

    @routes.route('/hostconfig', methods=['PUT'])
    def update_hostconfig(self, request):
        """
        Replace the device's host configuration.

        **Example request**:

        .. sourcecode:: http

           PUT /api/v1/config/hostconfig
           Content-Type: application/json

           {
             "firewall": {
               "defaults": {
                 "forward": "ACCEPT",
                 "input": "ACCEPT",
                 "output": "ACCEPT"
               }
             },
             ...
           }

        **Example response**:

        .. sourcecode:: http

           HTTP/1.1 200 OK
           Content-Type: application/json

           {
             change_id: 1
           }

        For a complete example, please see the Host Configuration section.
        """
        cors.config_cors(request)
        body = str2json(request.content.read())
        config = body['config']

        update = dict(updateClass='ROUTER',
                      updateType='sethostconfig',
                      name=constants.RESERVED_CHUTE,
                      tok=timeint(),
                      hostconfig=config)

        # We will return the change ID to the caller for tracking and log
        # retrieval.
        update['change_id'] = self.update_manager.assign_change_id()

        self.update_manager.add_update(**update)

        result = {'change_id': update['change_id']}
        request.setHeader('Content-Type', 'application/json')
        return json.dumps(result)

    @routes.route('/hostconfig', methods=['GET'])
    def get_hostconfig(self, request):
        """
        Get the device's current host configuration.

        **Example request**:

        .. sourcecode:: http

           GET /api/v1/config/hostconfig

        **Example response**:

        .. sourcecode:: http

           HTTP/1.1 200 OK
           Content-Type: application/json

           {
             "firewall": {
               "defaults": {
                 "forward": "ACCEPT",
                 "input": "ACCEPT",
                 "output": "ACCEPT"
               }
             },
             ...
           }

        For a complete example, please see the Host Configuration section.
        """
        cors.config_cors(request)
        config = hostconfig.prepareHostConfig()
        request.setHeader('Content-Type', 'application/json')
        return json.dumps(config, separators=(',', ':'))

    @routes.route('/new-config', methods=['GET'])
    def new_config(self, request):
        """
        Generate a new node configuration based on the hardware.

        **Example request**:

        .. sourcecode:: http

           GET /api/v1/config/new_config

        **Example response**:

        .. sourcecode:: http

           HTTP/1.1 200 OK
           Content-Type: application/json

           {
             "firewall": {
               "defaults": {
                 "forward": "ACCEPT",
                 "input": "ACCEPT",
                 "output": "ACCEPT"
               }
             },
             ...
           }

        For a complete example, please see the Host Configuration section.
        """
        cors.config_cors(request)
        config = hostconfig.prepareHostConfig(hostConfigPath='/dev/null')
        request.setHeader('Content-Type', 'application/json')
        return json.dumps(config, separators=(',', ':'))

    @routes.route('/pdid', methods=['GET'])
    def get_pdid(self, request):
        """
        Get the device's current ParaDrop ID. This is the identifier assigned
        by the cloud controller.

        **Example request**:

        .. sourcecode:: http

           GET /api/v1/config/pdid

        **Example response**:

        .. sourcecode:: http

           HTTP/1.1 200 OK
           Content-Type: application/json

           {
             pdid: "5890e1e5ab7e317e6c6e049f"
           }
        """
        cors.config_cors(request)
        pdid = nexus.core.info.pdid
        if pdid is None:
            pdid = ""
        request.setHeader('Content-Type', 'application/json')
        return json.dumps({'pdid': pdid})

    @routes.route('/provision', methods=['POST'])
    def provision(self, request):
        """
        Provision the device with credentials from a cloud controller.
        """
        cors.config_cors(request)
        body = str2json(request.content.read())
        routerId = body['routerId']
        apitoken = body['apitoken']
        pdserver = body['pdserver']
        wampRouter = body['wampRouter']

        changed = False
        if routerId != nexus.core.info.pdid \
            or pdserver != nexus.core.info.pdserver \
            or wampRouter != nexus.core.info.wampRouter:
            if pdserver and wampRouter:
                nexus.core.provision(routerId, pdserver, wampRouter)
            else:
                nexus.core.provision(routerId)
            changed = True

        if apitoken != nexus.core.getKey('apitoken'):
            nexus.core.saveKey(apitoken, 'apitoken')
            changed = True

        if changed:
            PDServerRequest.resetToken()
            nexus.core.jwt_valid = False

            def set_update_fetcher(session):
                session.set_update_fetcher(self.update_fetcher)

            @inlineCallbacks
            def start_polling(result):
                yield self.update_fetcher.start_polling()

            def send_response(result):
                response = dict()
                response['provisioned'] = True
                response['httpConnected'] = nexus.core.jwt_valid
                response['wampConnected'] = nexus.core.wamp_connected
                request.setHeader('Content-Type', 'application/json')
                return json.dumps(response)

            wampDeferred = nexus.core.connect(WampSession)
            wampDeferred.addCallback(set_update_fetcher)

            httpDeferred = sendStateReport()
            httpDeferred.addCallback(start_polling)

            identDeferred = sendNodeIdentity()

            dl = DeferredList([wampDeferred, httpDeferred, identDeferred],
                              consumeErrors=True)
            dl.addBoth(send_response)
            reactor.callLater(6, dl.cancel)
            return dl
        else:
            return json.dumps({
                'success':
                False,
                'message':
                'No change on the provision parameters'
            })

    @routes.route('/provision', methods=['GET'])
    def get_provision(self, request):
        """
        Get the provision status of the device.
        """
        cors.config_cors(request)
        result = dict()
        result['routerId'] = nexus.core.info.pdid
        result['pdserver'] = nexus.core.info.pdserver
        result['wampRouter'] = nexus.core.info.wampRouter
        apitoken = nexus.core.getKey('apitoken')
        result['provisioned'] = (result['routerId'] is not None and \
                                 apitoken is not None)
        result['httpConnected'] = nexus.core.jwt_valid
        result['wampConnected'] = nexus.core.wamp_connected
        request.setHeader('Content-Type', 'application/json')
        return json.dumps(result)

    @routes.route('/settings', methods=['GET'])
    def get_settings(self, request):
        """
        Get current values of system settings.

        These are the values from paradrop.base.settings. Settings are loaded
        at system initialization from the settings.ini file and environment
        variables. They are intended to be read-only after initialization.

        This endpoint returns the settings as a dictionary with lowercase field
        names.

        Example:
        {
            "portal_server_port": 8080,
            ...
        }
        """
        cors.config_cors(request)
        request.setHeader('Content-Type', 'application/json')

        result = {}
        for name, value in settings.iterate_module_attributes(settings):
            result[name.lower()] = value

        return json.dumps(result)

    @routes.route('/startUpdate', methods=['POST'])
    def start_update(self, request):
        cors.config_cors(request)
        self.update_manager.startUpdate()
        request.setHeader('Content-Type', 'application/json')
        return json.dumps({'success': True})

    @routes.route('/factoryReset', methods=['POST'])
    @inlineCallbacks
    def factory_reset(self, request):
        """
        Initiate the factory reset process.
        """
        cors.config_cors(request)
        update = dict(updateClass='ROUTER',
                      updateType='factoryreset',
                      name='PARADROP',
                      tok=timeint())
        update = yield self.update_manager.add_update(**update)
        returnValue(json.dumps(update.result))

    @routes.route('/pdconf', methods=['GET'])
    def pdconf(self, request):
        """
        Get configuration sections from pdconf.

        This returns a list of configuration sections and whether they were
        successfully applied. This is intended for debugging purposes.
        """
        cors.config_cors(request)
        request.setHeader('Content-Type', 'application/json')
        return pdconf_client.systemStatus()

    @routes.route('/pdconf', methods=['PUT'])
    def pdconf_reload(self, request):
        """
        Trigger pdconf to reload UCI configuration files.

        Trigger pdconf to reload UCI configuration files and return the status.
        This function is intended for low-level debugging of the paradrop
        pdconf module.
        """
        cors.config_cors(request)
        request.setHeader('Content-Type', 'application/json')
        return pdconf_client.reloadAll()

    @routes.route('/sshKeys/<user>', methods=['GET', 'POST'])
    def sshKeys(self, request, user):
        """
        Manage list of authorized keys for SSH access.
        """
        cors.config_cors(request)
        request.setHeader('Content-Type', 'application/json')

        if request.method == "GET":
            try:
                keys = ssh_keys.getAuthorizedKeys(user)
                return json.dumps(keys)
            except Exception as e:
                out.warn(str(e))
                request.setResponseCode(404)
                return json.dumps({'message': str(e)})
        else:
            body = str2json(request.content.read())
            key = body['key'].strip()

            try:
                ssh_keys.addAuthorizedKey(key, user)
                return json.dumps(body)
            except Exception as e:
                out.warn(str(e))
                request.setResponseCode(404)
                return json.dumps({'message': str(e)})
Beispiel #33
0
class MyWebMail(object):
    app = Klein()
    buckets = {}

    def __init__(self):
        self.init_stuff()

    def init_stuff(self):
        cli = riak.RiakClient()
        self.buckets['cred'] = cli.bucket('cred')
        self.buckets['mailmeta'] = cli.bucket('mailmeta')
        self.buckets['mailbody'] = cli.bucket('mailbody')

    @inlineCallbacks
    def getcred(self, key):
        obj = yield self.buckets['cred'].get(key)
        returnValue(obj.get_data())

    # user mgmt here.
    @app.route("/profile", methods=['GET'])
    @inlineCallbacks
    def modifyuser(self, request):
        login = request.getUser()
        curdata = yield self.getcred(login)
        mod_dsl = dsl.replace('NewEmailUser', 'ModifyUser')
        d = curdata
        d.update({'login': login})
        myform = myhtml.renderForm(myhtml.parse(mod_dsl),
                                   action="/profile",
                                   method="POST")
        header = '<html><head><title>new user</title>' + '<style type="text/css" media="screen">' + mystyle2 + '</style></head>'
        request.write(header +
                      '<body><h3>Modify User</h3><div id="container">')
        form2 = str(myform).replace('NONE', '').replace(
            '<label>Login',
            '<fieldset><legend>User info</legend><div class="fm-req"><label>Login'
        )
        form2 = form2.replace('</form>', '</div></div></form>')

        tmp = str(form2) % d
        request.write(tmp.encode('ascii'))
        request.write('</body></html>')

    @app.route("/profile", methods=['POST'])
    @inlineCallbacks
    def modifyuser_post(self, request):
        login = request.getUser()
        a = request.args
        data = yield self.getcred(login)
        name = a.get('name')[0]
        password = a.get('password')[0]
        # check!
        if len(password) < 8:
            request.write(
                '<b>Password was too short! need min. 8 characters.</b>')
        else:
            data.update({
                "name": name,
                "password": password,
                "timestamp": int(time.time())
            })
            obj = self.buckets['cred'].new(login, data)
            yield obj.store()
            print('saved modified entry to users!')
            request.write('Modified user %s. ' % (login))
        request.write('<p><a href="/profile">Return.</a>')

    # webmail routes here.
    @app.route('/')
    def index(self, request):
        return 'it works...'

    @app.route('/search', methods=['GET'])
    def search(self, request):
        out = '''
<h2>full text search</h2>
<p>
type your search term(s) here:
<form action="/search" method="POST" >
<input type="TEXT" name="terms" value="hae..." />
<input type="SUBMIT" name="SUBMIT" />
</form>
'''
        request.write(out)

    @app.route('/search', methods=['POST'])
    @inlineCallbacks
    def do_search(self, request):
        import quopri, re
        a = request.args
        login = request.getUser()
        obj = yield self.buckets['mailmeta'].get(login)
        user_emails = obj.get_data().keys()
        terms = a.get('terms')[0]
        print 'TERMS:', terms
        res = yield self.buckets['mailbody'].search(terms)
        request.write('search found this:')
        r1 = re.compile(r".*Delivered-To:\s([a-zA-Z0-9.-]+)@.*?\n.*",
                        re.DOTALL)  # is this all?
        for doc in res['docs']:
            data = doc['value']
            m = r1.match(data)
            if m.group(1) != login:
                continue  # this msg was not for this user, skip it.
            content_trans = 'Content-Transfer-Encoding: quoted-printable'
            if content_trans in data:
                headers, _, body = data.partition(content_trans)
            else:
                headers, body = data, ''
            out = headers.decode('ascii')  # headers should always be ascii-onl
            s = quopri.decodestring(body).decode('utf-8', errors='replace')
            print type(out), type(s)
            out = out + s
            out = out.encode('utf-8')
            request.write('<pre>' + out + '</pre>')

    @app.route('/showSess')
    def showSess(self, request):
        s = request.getSession()
        print request.getUser()
        print 'DEBUG:', s.sessionNamespaces, dir(s.sessionNamespaces)
        return 'Session id is: ' + s.uid

    @app.route('/expireSess')
    def expSess(self, request):
        request.getSession().expire()
        return 'Session expired.'

    @app.route('/inbox')
    @defer.inlineCallbacks
    def inbox(self, request):
        user = request.getUser()
        obj = yield self.buckets['mailmeta'].get(user)
        data = obj.get_data()
        print 'DEBUG:', data, type(data)
        out1 = 'Num of mails: %d' % len(data)
        out2 = 'Total bytes used: %d' % sum([data[x]['size'] for x in data])
        request.write('<p>'.join([out1, out2]).encode('ascii'))

        # list all mail headers

        for mailid in data:
            request.write('<p>')
            obj = yield self.buckets['mailbody'].get(mailid)
            body = obj.get_data()
            #print body, type(body)
            lst = body.split('\n')
            out = []
            for line in lst:
                if line.startswith('From:') or line.startswith(
                        'To:') or line.startswith('Subject:'):
                    out.append(line)
            request.write('<br>'.join(out))
Beispiel #34
0
class ChuteApi(object):
    routes = Klein()

    def __init__(self, update_manager):
        self.update_manager = update_manager

    @routes.route('/', methods=['GET'])
    def get_chutes(self, request):
        """
        List installed chutes.

        **Example request**:

        .. sourcecode:: http

           GET /api/v1/chutes/

        **Example response**:

        .. sourcecode:: http

           HTTP/1.1 200 OK
           Content-Type: application/json

           [
             {
               "environment": {},
               "name": "hello-world",
               "allocation": {
                 "cpu_shares": 1024,
                 "prioritize_traffic": false
               },
               "state": "running",
               "version": "x1511808778",
               "resources": null
             }
           ]
        """
        cors.config_cors(request)
        request.setHeader('Content-Type', 'application/json')

        chuteStorage = ChuteStorage()
        chutes = chuteStorage.getChuteList()
        allocation = resource.computeResourceAllocation(chutes)

        result = []
        for chute in chutes:
            container = ChuteContainer(chute.name)

            result.append({
                'name': chute.name,
                'state': container.getStatus(),
                'version': getattr(chute, 'version', None),
                'allocation': allocation.get(chute.name, None),
                'environment': getattr(chute, 'environment', None),
                'resources': getattr(chute, 'resources', None)
            })

        return json.dumps(result)

    @routes.route('/', methods=['POST'])
    def create_chute(self, request):
        cors.config_cors(request)

        update = dict(updateClass='CHUTE',
                      updateType='create',
                      tok=pdutils.timeint())

        ctype = request.requestHeaders.getRawHeaders('Content-Type',
                                                     default=[None])[0]
        if ctype == "application/x-tar":
            workdir, paradrop_yaml = extract_tarred_chute(request.content)
            config = paradrop_yaml.get("config", {})

            # Try to read chute name from top level (preferred) or from config
            # object (deprecated).
            if 'name' in paradrop_yaml:
                update['name'] = paradrop_yaml['name']
            elif 'name' in config:
                out.warn(
                    "Deprecated: move chute name to top level of config file.")
                update['name'] = config['name']
            else:
                raise Exception("Chute name not found in configuration file.")

            update['workdir'] = workdir
            chute_version = paradrop_yaml.get("version", None)
            update['version'] = "x{}".format(update['tok'])
            update.update(config)
        else:
            # TODO: this case is not tested
            body = json.loads(request.content.read())
            config = body['config']
            update.update(config)
            # Set a time-based version number for side-loaded chutes because we do
            # not expect they to receive it from the config file.
            update['version'] = "x{}".format(update['tok'])

        # We will return the change ID to the caller for tracking and log
        # retrieval.
        update['change_id'] = self.update_manager.assign_change_id()

        d = self.update_manager.add_update(**update)

        result = {'change_id': update['change_id']}
        request.setHeader('Content-Type', 'application/json')
        return json.dumps(result)

    @routes.route('/<chute>', methods=['GET'])
    def get_chute(self, request, chute):
        """
        Get information about an installed chute.

        **Example request**:

        .. sourcecode:: http

           GET /api/v1/chutes/hello-world

        **Example response**:

        .. sourcecode:: http

           HTTP/1.1 200 OK
           Content-Type: application/json

           {
             "environment": {},
             "name": "hello-world",
             "allocation": {
               "cpu_shares": 1024,
               "prioritize_traffic": false
             },
             "state": "running",
             "version": "x1511808778",
             "resources": null
           }
        """
        cors.config_cors(request)

        request.setHeader('Content-Type', 'application/json')

        chute_obj = ChuteStorage.chuteList[chute]
        container = ChuteContainer(chute)

        chuteStorage = ChuteStorage()
        allocation = resource.computeResourceAllocation(
            chuteStorage.getChuteList())

        result = {
            'name': chute,
            'state': container.getStatus(),
            'version': getattr(chute_obj, 'version', None),
            'allocation': allocation.get(chute, None),
            'environment': getattr(chute_obj, 'environment', None),
            'resources': getattr(chute_obj, 'resources', None)
        }

        return json.dumps(result)

    @routes.route('/<chute>', methods=['PUT'])
    def update_chute(self, request, chute):
        cors.config_cors(request)

        update = dict(updateClass='CHUTE',
                      updateType='update',
                      tok=pdutils.timeint(),
                      name=chute)

        ctype = request.requestHeaders.getRawHeaders('Content-Type',
                                                     default=[None])[0]
        if ctype == "application/x-tar":
            workdir, paradrop_yaml = extract_tarred_chute(request.content)
            config = paradrop_yaml.get("config", {})

            # Try to read chute name from top level (preferred) or from config
            # object (deprecated).
            if 'name' in paradrop_yaml:
                update['name'] = paradrop_yaml['name']
            elif 'name' in config:
                out.warn(
                    "Deprecated: move chute name to top level of config file.")
                update['name'] = config['name']
            else:
                raise Exception("Chute name not found in configuration file.")

            update['workdir'] = workdir
            chute_version = paradrop_yaml.get("version", None)
            update['version'] = "x{}".format(update['tok'])
            update.update(config)
        else:
            body = json.loads(request.content.read())
            config = body['config']

            update.update(config)

        # Set a time-based version number for side-loaded chutes because we do
        # not expect the to receive it from the config file.
        update['version'] = "x{}".format(update['tok'])

        # We will return the change ID to the caller for tracking and log
        # retrieval.
        update['change_id'] = self.update_manager.assign_change_id()

        d = self.update_manager.add_update(**update)

        result = {'change_id': update['change_id']}
        request.setHeader('Content-Type', 'application/json')
        return json.dumps(result)

    @routes.route('/<chute>', methods=['DELETE'])
    def delete_chute(self, request, chute):
        cors.config_cors(request)

        update = dict(updateClass='CHUTE',
                      updateType='delete',
                      tok=pdutils.timeint(),
                      name=chute)

        # We will return the change ID to the caller for tracking and log
        # retrieval.
        update['change_id'] = self.update_manager.assign_change_id()

        d = self.update_manager.add_update(**update)

        result = {'change_id': update['change_id']}
        request.setHeader('Content-Type', 'application/json')
        return json.dumps(result)

    @routes.route('/<chute>/stop', methods=['POST'])
    def stop_chute(self, request, chute):
        cors.config_cors(request)

        update = dict(updateClass='CHUTE',
                      updateType='stop',
                      tok=pdutils.timeint(),
                      name=chute)

        # We will return the change ID to the caller for tracking and log
        # retrieval.
        update['change_id'] = self.update_manager.assign_change_id()

        d = self.update_manager.add_update(**update)

        result = {'change_id': update['change_id']}
        request.setHeader('Content-Type', 'application/json')
        return json.dumps(result)

    @routes.route('/<chute>/start', methods=['POST'])
    def start_chute(self, request, chute):
        cors.config_cors(request)

        update = dict(updateClass='CHUTE',
                      updateType='start',
                      tok=pdutils.timeint(),
                      name=chute)

        try:
            body = json.loads(request.content.read())

            # Chute environment variables can be replaced during the operation.
            update['environment'] = body['environment']
        except Exception as error:
            pass

        # We will return the change ID to the caller for tracking and log
        # retrieval.
        update['change_id'] = self.update_manager.assign_change_id()

        d = self.update_manager.add_update(**update)

        result = {'change_id': update['change_id']}
        request.setHeader('Content-Type', 'application/json')
        return json.dumps(result)

    @routes.route('/<chute>/restart', methods=['POST'])
    def restart_chute(self, request, chute):
        cors.config_cors(request)

        update = dict(updateClass='CHUTE',
                      updateType='restart',
                      tok=pdutils.timeint(),
                      name=chute)

        try:
            body = json.loads(request.content.read())

            # Chute environment variables can be replaced during the operation.
            update['environment'] = body['environment']
        except Exception as error:
            pass

        # We will return the change ID to the caller for tracking and log
        # retrieval.
        update['change_id'] = self.update_manager.assign_change_id()

        d = self.update_manager.add_update(**update)

        result = {'change_id': update['change_id']}
        request.setHeader('Content-Type', 'application/json')
        return json.dumps(result)

    @routes.route('/<chute>/cache', methods=['GET'])
    def get_chute_cache(self, request, chute):
        """
        Get chute cache contents.

        The chute cache is a key-value store used during chute installation.
        It can be useful for debugging the Paradrop platform.
        """
        cors.config_cors(request)

        request.setHeader('Content-Type', 'application/json')

        chute_obj = ChuteStorage.chuteList[chute]
        result = chute_obj.getCacheContents()

        return json.dumps(result, cls=ChuteCacheEncoder)

    @routes.route('/<chute>/config', methods=['GET'])
    def get_chute_config(self, request, chute):
        """
        Get current chute configuration.

        **Example request**:

        .. sourcecode:: http

           GET /api/v1/chutes/captive-portal/config

        **Example response**:

        .. sourcecode:: http

           HTTP/1.1 200 OK
           Content-Type: application/json

           {
             "net": {
               "wifi": {
                 "dhcp": {
                   "lease": "1h",
                   "limit": 250,
                   "start": 3
                 },
                 "intfName": "wlan0",
                 "options": {
                   "isolate": True
                 },
                 "ssid": "Free WiFi",
                 "type": "wifi"
               }
             }
           }
        """
        cors.config_cors(request)
        request.setHeader('Content-Type', 'application/json')

        chute_obj = ChuteStorage.chuteList[chute]
        config = chute_obj.getConfiguration()

        return json.dumps(config)

    @routes.route('/<chute>/config', methods=['PUT'])
    def set_chute_config(self, request, chute):
        """
        Update the chute configuration and restart to apply changes.

        **Example request**:

        .. sourcecode:: http

           PUT /api/v1/chutes/captive-portal/config
           Content-Type: application/json

           {
             "net": {
               "wifi": {
                 "dhcp": {
                   "lease": "1h",
                   "limit": 250,
                   "start": 3
                 },
                 "intfName": "wlan0",
                 "options": {
                   "isolate": True
                 },
                 "ssid": "Better Free WiFi",
                 "type": "wifi"
               }
             }
           }

        **Example response**:

        .. sourcecode:: http

           HTTP/1.1 200 OK
           Content-Type: application/json

           {
             "change_id": 1
           }
        """
        cors.config_cors(request)
        request.setHeader('Content-Type', 'application/json')

        update = dict(updateClass='CHUTE',
                      updateType='restart',
                      tok=pdutils.timeint(),
                      name=chute)

        try:
            body = json.loads(request.content.read())
            update.update(body)
        except Exception as error:
            pass

        # We will return the change ID to the caller for tracking and log
        # retrieval.
        update['change_id'] = self.update_manager.assign_change_id()

        d = self.update_manager.add_update(**update)

        result = {'change_id': update['change_id']}
        return json.dumps(result)

    @routes.route('/<chute>/networks', methods=['GET'])
    def get_networks(self, request, chute):
        """
        Get list of networks configured for the chute.

        **Example request**:

        .. sourcecode:: http

           GET /api/v1/chutes/captive-portal/networks

        **Example response**:

        .. sourcecode:: http

           HTTP/1.1 200 OK
           Content-Type: application/json

           [
             {
               "interface": "wlan0",
               "type": "wifi",
               "name": "wifi"
             }
           ]
        """
        cors.config_cors(request)

        request.setHeader('Content-Type', 'application/json')

        chute_obj = ChuteStorage.chuteList[chute]
        networkInterfaces = chute_obj.getCache('networkInterfaces')

        result = []
        for iface in networkInterfaces:
            data = {
                'name': iface['name'],
                'type': iface['netType'],
                'interface': iface['internalIntf']
            }
            result.append(data)

        return json.dumps(result)

    @routes.route('/<chute>/networks/<network>', methods=['GET'])
    def get_network(self, request, chute, network):
        """
        Get information about a network configured for the chute.

        **Example request**:

        .. sourcecode:: http

           GET /api/v1/chutes/captive-portal/networks/wifi

        **Example response**:

        .. sourcecode:: http

           HTTP/1.1 200 OK
           Content-Type: application/json

           {
             "interface": "wlan0",
             "type": "wifi",
             "name": "wifi"
           }
        """
        cors.config_cors(request)

        request.setHeader('Content-Type', 'application/json')

        chute_obj = ChuteStorage.chuteList[chute]
        networkInterfaces = chute_obj.getCache('networkInterfaces')

        data = {}
        for iface in networkInterfaces:
            if iface['name'] != network:
                continue

            data = {
                'name': iface['name'],
                'type': iface['netType'],
                'interface': iface['internalIntf']
            }

        return json.dumps(data)

    @routes.route('/<chute>/networks/<network>/leases', methods=['GET'])
    def get_leases(self, request, chute, network):
        """
        Get current list of DHCP leases for chute network.

        Returns a list of DHCP lease records with the following fields:

        expires
          lease expiration time (seconds since Unix epoch)
        mac_addr
          device MAC address
        ip_addr
          device IP address
        hostname
          name that the device reported
        client_id
          optional identifier supplied by device

        **Example request**:

        .. sourcecode:: http

           GET /api/v1/chutes/captive-portal/networks/wifi/leases

        **Example response**:

        .. sourcecode:: http

           HTTP/1.1 200 OK
           Content-Type: application/json

           [
             {
               "client_id": "01:5c:59:48:7d:b9:e6",
               "expires": "1511816276",
               "ip_addr": "192.168.128.64",
               "mac_addr": "5c:59:48:7d:b9:e6",
               "hostname": "paradrops-iPod"
             }
           ]
        """
        cors.config_cors(request)

        request.setHeader('Content-Type', 'application/json')

        chute_obj = ChuteStorage.chuteList[chute]
        externalSystemDir = chute_obj.getCache('externalSystemDir')

        leasefile = 'dnsmasq-{}.leases'.format(network)
        path = os.path.join(externalSystemDir, leasefile)

        # The format of the dnsmasq leases file is one entry per line with
        # space-separated fields.
        keys = ['expires', 'mac_addr', 'ip_addr', 'hostname', 'client_id']

        leases = []
        with open(path, 'r') as source:
            for line in source:
                parts = line.strip().split()
                leases.append(dict(zip(keys, parts)))

        return json.dumps(leases)

    @routes.route('/<chute>/networks/<network>/ssid', methods=['GET'])
    def get_ssid(self, request, chute, network):
        """
        Get currently configured SSID for the chute network.

        **Example request**:

        .. sourcecode:: http

           GET /api/v1/chutes/captive-portal/networks/wifi/ssid

        **Example response**:

        .. sourcecode:: http

           HTTP/1.1 200 OK
           Content-Type: application/json

           {
             "ssid": "Free WiFi",
             "bssid": "02:00:08:24:03:dd"
           }
        """
        cors.config_cors(request)
        request.setHeader('Content-Type', 'application/json')

        chute_obj = ChuteStorage.chuteList[chute]
        networkInterfaces = chute_obj.getCache('networkInterfaces')

        ifname = None
        for iface in networkInterfaces:
            if iface['name'] == network:
                ifname = iface['externalIntf']
                break

        address = os.path.join(settings.PDCONFD_WRITE_DIR, "hostapd", ifname)
        return hostapd_control.execute(address, command="GET_CONFIG")

    @routes.route('/<chute>/networks/<network>/ssid', methods=['PUT'])
    def set_ssid(self, request, chute, network):
        """
        Change the configured SSID for the chute network.

        The change will not persist after a reboot. If a persistent change is
        desired, you should update the chute configuration instead.

        **Example request**:

        .. sourcecode:: http

           PUT /api/v1/chutes/captive-portal/networks/wifi/ssid
           Content-Type: application/json

           {
             "ssid": "Best Free WiFi"
           }

        **Example response**:

        .. sourcecode:: http

           HTTP/1.1 200 OK
           Content-Type: application/json

           {
             "message": "OK"
           }
        """
        cors.config_cors(request)
        request.setHeader('Content-Type', 'application/json')

        chute_obj = ChuteStorage.chuteList[chute]
        networkInterfaces = chute_obj.getCache('networkInterfaces')

        ifname = None
        for iface in networkInterfaces:
            if iface['name'] == network:
                ifname = iface['externalIntf']
                break

        body = json.loads(request.content.read())
        if "ssid" not in body:
            raise Exception("ssid required")

        command = "SET ssid {}".format(body['ssid'])
        address = os.path.join(settings.PDCONFD_WRITE_DIR, "hostapd", ifname)
        return hostapd_control.execute(address, command=command)

    @routes.route('/<chute>/networks/<network>/hostapd_status',
                  methods=['GET'])
    def get_hostapd_status(self, request, chute, network):
        """
        Get low-level status information from the access point.

        **Example request**:

        .. sourcecode:: http

           GET /api/v1/chutes/captive-portal/networks/wifi/hostapd_status

        **Example response**:

        .. sourcecode:: http

           HTTP/1.1 200 OK
           Content-Type: application/json

           {
             "olbc_ht": "0",
             "cac_time_left_seconds": "N/A",
             "num_sta_no_short_slot_time": "0",
             "olbc": "1",
             "num_sta_non_erp": "0",
             "ht_op_mode": "0x4",
             "state": "ENABLED",
             "num_sta_ht40_intolerant": "0",
             "channel": "11",
             "bssid[0]": "02:00:08:24:03:dd",
             "ieee80211n": "1",
             "cac_time_seconds": "0",
             "num_sta[0]": "1",
             "ieee80211ac": "0",
             "phy": "phy0",
             "num_sta_ht_no_gf": "1",
             "freq": "2462",
             "num_sta_ht_20_mhz": "1",
             "num_sta_no_short_preamble": "0",
             "secondary_channel": "0",
             "ssid[0]": "Free WiFi",
             "num_sta_no_ht": "0",
             "bss[0]": "vwlan7e1b"
           }
        """
        cors.config_cors(request)
        request.setHeader('Content-Type', 'application/json')

        chute_obj = ChuteStorage.chuteList[chute]
        networkInterfaces = chute_obj.getCache('networkInterfaces')

        ifname = None
        for iface in networkInterfaces:
            if iface['name'] == network:
                ifname = iface['externalIntf']
                break

        address = os.path.join(settings.PDCONFD_WRITE_DIR, "hostapd", ifname)
        return hostapd_control.execute(address, command="STATUS")

    @routes.route('/<chute>/networks/<network>/stations', methods=['GET'])
    def get_stations(self, request, chute, network):
        """
        Get detailed information about connected wireless stations.

        **Example request**:

        .. sourcecode:: http

           GET /api/v1/chutes/captive-portal/networks/wifi/stations

        **Example response**:

        .. sourcecode:: http

           HTTP/1.1 200 OK
           Content-Type: application/json

           [
             {
               "rx_packets": "230",
               "tdls_peer": "no",
               "authenticated": "yes",
               "rx_bytes": "12511",
               "tx_bitrate": "1.0 MBit/s",
               "tx_retries": "0",
               "signal": "-45 [-49, -48] dBm",
               "authorized": "yes",
               "rx_bitrate": "65.0 MBit/s MCS 7",
               "mfp": "no",
               "tx_failed": "0",
               "inactive_time": "4688 ms",
               "mac_addr": "5c:59:48:7d:b9:e6",
               "tx_bytes": "34176",
               "wmm_wme": "yes",
               "preamble": "short",
               "tx_packets": "88",
               "signal_avg": "-44 [-48, -47] dBm"
             }
           ]
        """
        cors.config_cors(request)

        request.setHeader('Content-Type', 'application/json')

        chute_obj = ChuteStorage.chuteList[chute]
        networkInterfaces = chute_obj.getCache('networkInterfaces')

        ifname = None
        for iface in networkInterfaces:
            if iface['name'] == network:
                ifname = iface['externalIntf']
                break

        cmd = ['iw', 'dev', ifname, 'station', 'dump']
        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)

        stations = []
        current = {}

        for line in proc.stdout:
            line = line.strip()

            match = re.match("Station\s+(\S+)\s+.*", line)
            if match is not None:
                current = {'mac_addr': match.group(1)}
                stations.append(current)
                continue

            match = re.match("(.*):\s+(.*)", line)
            if match is not None:
                key = match.group(1).lower().replace(' ',
                                                     '_').replace('/', '_')
                current[key] = match.group(2)

        return json.dumps(stations)

    @routes.route('/<chute>/networks/<network>/stations/<mac>',
                  methods=['GET'])
    def get_station(self, request, chute, network, mac):
        """
        Get detailed information about a connected station.

        **Example request**:

        .. sourcecode:: http

           GET /api/v1/chutes/captive-portal/networks/wifi/stations/5c:59:48:7d:b9:e6

        **Example response**:

        .. sourcecode:: http

           HTTP/1.1 200 OK
           Content-Type: application/json

           {
             "rx_packets": "230",
             "tdls_peer": "no",
             "authenticated": "yes",
             "rx_bytes": "12511",
             "tx_bitrate": "1.0 MBit/s",
             "tx_retries": "0",
             "signal": "-45 [-49, -48] dBm",
             "authorized": "yes",
             "rx_bitrate": "65.0 MBit/s MCS 7",
             "mfp": "no",
             "tx_failed": "0",
             "inactive_time": "4688 ms",
             "mac_addr": "5c:59:48:7d:b9:e6",
             "tx_bytes": "34176",
             "wmm_wme": "yes",
             "preamble": "short",
             "tx_packets": "88",
             "signal_avg": "-44 [-48, -47] dBm"
           }
        """
        cors.config_cors(request)

        request.setHeader('Content-Type', 'application/json')

        chute_obj = ChuteStorage.chuteList[chute]
        networkInterfaces = chute_obj.getCache('networkInterfaces')

        ifname = None
        for iface in networkInterfaces:
            if iface['name'] == network:
                ifname = iface['externalIntf']
                break

        cmd = ['iw', 'dev', ifname, 'station', 'get', mac]
        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)

        station = {}

        for line in proc.stdout:
            line = line.strip()

            match = re.match("Station\s+(\S+)\s+.*", line)
            if match is not None:
                station['mac_addr'] = match.group(1)
                continue

            match = re.match("(.*):\s+(.*)", line)
            if match is not None:
                key = match.group(1).lower().replace(' ',
                                                     '_').replace('/', '_')
                station[key] = match.group(2)

        return json.dumps(station)

    @routes.route('/<chute>/networks/<network>/stations/<mac>',
                  methods=['DELETE'])
    def delete_station(self, request, chute, network, mac):
        cors.config_cors(request)

        request.setHeader('Content-Type', 'application/json')

        chute_obj = ChuteStorage.chuteList[chute]
        networkInterfaces = chute_obj.getCache('networkInterfaces')

        ifname = None
        for iface in networkInterfaces:
            if iface['name'] == network:
                ifname = iface['externalIntf']
                break

        cmd = ['iw', 'dev', ifname, 'station', 'del', mac]
        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)

        messages = []
        for line in proc.stdout:
            line = line.strip()
            messages.append(line)

        return json.dumps(messages)

    @routes.route('/<chute>/networks/<network>/hostapd_control/ws',
                  branch=True,
                  methods=['GET'])
    def hostapd_control(self, request, chute, network):
        chute_obj = ChuteStorage.chuteList[chute]
        networkInterfaces = chute_obj.getCache('networkInterfaces')

        ifname = None
        for iface in networkInterfaces:
            if iface['name'] == network:
                ifname = iface['externalIntf']
                break

        ctrl_iface = os.path.join(settings.PDCONFD_WRITE_DIR, "hostapd",
                                  ifname)
        factory = hostapd_control.HostapdControlWSFactory(ctrl_iface)
        factory.setProtocolOptions(autoPingInterval=10, autoPingTimeout=5)
        return WebSocketResource(factory)
Beispiel #35
0
                    id="ltri-dota-gsi-money",
                    desc="Machine State for DOTA Money"),
            "dota-gsi-money-inv":
            Machine(name="dota-GSI",
                    iname="MONEY_INV",
                    id="ltri-dota-gsi-money-inv",
                    desc="Machine State for DOTA Money (inverted)"),
        },
        speed_enum={"FHUNDREDTHS": .05})
    # lol
    serialized = json.loads(
        json.dumps(schema, indent=4, default=pydantic_encoder))
    return serialized


webapp = Klein()


@webapp.route('/', methods=['POST'])
def index(request: Request):
    content = json.loads(request.content.read())
    try:
        new_state = GameState(**content)

    except Exception as e:
        print(content)
        print(e)
        return "wew"

    wamp_component.state = new_state
    print(wamp_component.state.map.game_state)
Beispiel #36
0
class AuthApi(object):
    routes = Klein()

    def __init__(self, password_manager, token_manager):
        self.password_manager = password_manager
        self.token_manager = token_manager

    @routes.route('/local', methods=['POST'])
    def local_login(self, request):
        """
        Login using local authentication (username+password).
        """
        cors.config_cors(request)
        request.setHeader('Content-Type', 'application/json')

        body = json.loads(request.content.read())
        username = body.get('username', self.password_manager.DEFAULT_USER_NAME)
        password = body.get('password', self.password_manager.DEFAULT_PASSWORD)

        success = self.password_manager.verify_password(username, password)
        result = {
            'username': username,
            'success': success
        }

        if success:
            token = self.token_manager.issue(username, domain="localhost",
                    role="admin")
            result['token'] = token
        else:
            request.setResponseCode(401)

        return json.dumps(result)

    @routes.route('/cloud', methods=['POST'])
    def auth_cloud(self, request):
        """
        Login using credentials from the cloud controller.

        This is an experimental new login method that lets users
        present a token that they received from the cloud controller
        as a login credential for a node. The idea is to enable easy
        access for multiple developers to share a node, for example,
        during a tutorial.

        Instead of a username/password, the user presents a token received
        from the cloud controller. The verify_cloud_token function
        verifies the validity of the token with the controller, and if
        successful, retrieves information about the bearer, particularly
        the username and role. Finally, we generate a new token that
        enables the user to authenticate with local API endpoints.
        """
        cors.config_cors(request)
        request.setHeader('Content-Type', 'application/json')

        body = json.loads(request.content.read())
        token = body.get('token', '')

        headers = {
            "Authorization": "Bearer {}".format(token)
        }
        node_id = nexus.core.info.pdid

        # Pass custom Authorization header and setAuthHeader=False to prevent
        # PDServerRequest from using the node's own authorization token.
        d1 = PDServerRequest("/api/users/me",
                headers=headers,
                setAuthHeader=False
            ).get()
        d2 = PDServerRequest("/api/routers/{}".format(node_id),
                headers=headers,
                setAuthHeader=False
            ).get()

        def token_verified(responses):
            for response in responses:
                if not response.success or response.data is None:
                    request.setResponseCode(401)
                    return

            remote_user = responses[0].data
            node_info = responses[1].data

            role = get_access_level(remote_user, node_info)

            token = self.token_manager.issue(remote_user['email'],
                    domain="paradrop.org", role=role)

            result = {
                'token': token,
                'username': remote_user['email']
            }
            return json.dumps(result)

        d = defer.gatherResults([d1, d2])
        d.addCallback(token_verified)
        return d
Beispiel #37
0
class ProxyApp(object):
    app = Klein()
    ns = "{http://www.yale.edu/tp/cas}"
    port = None
    logout_instant_skew = 5
    ticket_name = 'ticket'
    service_name = 'service'
    renew_name = 'renew'
    pgturl_name = 'pgtUrl'
    reactor = reactor
    auth_info_resource = None
    auth_info_callback = None
    remoteUserHeader = 'Remote-User'
    logout_patterns = None
    logout_passthrough = False
    verbose = False
    proxy_client_endpoint_s = None
    cas_client_endpoint_s = None

    def __init__(self,
                 proxied_url,
                 cas_info,
                 fqdn=None,
                 authorities=None,
                 plugins=None,
                 is_https=True,
                 excluded_resources=None,
                 excluded_branches=None,
                 remote_user_header=None,
                 logout_patterns=None,
                 logout_passthrough=False,
                 template_dir=None,
                 template_resource='/_templates',
                 proxy_client_endpoint_s=None,
                 cas_client_endpoint_s=None):
        self.proxy_client_endpoint_s = proxy_client_endpoint_s
        self.cas_client_endpoint_s = cas_client_endpoint_s
        self.logout_passthrough = logout_passthrough
        self.template_dir = template_dir
        if template_dir is not None:
            self.template_loader_ = FileSystemLoader(template_dir)
            self.template_env_ = Environment()
            self.templateStaticResource_ = self.create_template_static_resource(
            )
        if template_resource is not None:
            if not template_resource.endswith('/'):
                template_resource = "{0}/".format(template_resource)
        if template_resource is not None and template_dir is not None:
            static_base = "{0}static/".format(template_resource)
            self.static = self.app.route(static_base,
                                         branch=True)(self.__class__.static)
            self.static_base = static_base
        self.template_resource = template_resource
        if logout_patterns is not None:
            self.logout_patterns = [
                parse_url_pattern(pattern) for pattern in logout_patterns
            ]
        for pattern in self.logout_patterns:
            assert pattern is None or pattern.scheme == '', (
                "Logout pattern '{0}' must be a relative URL.".format(pattern))
        if remote_user_header is not None:
            self.remoteUserHeader = remote_user_header
        self.excluded_resources = excluded_resources
        self.excluded_branches = excluded_branches
        self.is_https = is_https
        if proxied_url.endswith('/'):
            proxied_url = proxied_url[:-1]
        self.proxied_url = proxied_url
        p = urlparse.urlparse(proxied_url)
        self.p = p
        self.proxied_scheme = p.scheme
        netloc = p.netloc
        self.proxied_netloc = netloc
        self.proxied_host = netloc.split(':')[0]
        self.proxied_path = p.path
        self.cas_info = cas_info
        cas_param_names = set([])
        cas_param_names.add(self.ticket_name.lower())
        cas_param_names.add(self.service_name.lower())
        cas_param_names.add(self.renew_name.lower())
        cas_param_names.add(self.pgturl_name.lower())
        self.cas_param_names = cas_param_names
        if fqdn is None:
            fqdn = socket.getfqdn()
        self.fqdn = fqdn
        self.valid_sessions = {}
        self.logout_tickets = {}
        self._make_agents(authorities)
        # Sort/tag plugins
        if plugins is None:
            plugins = []
        content_modifiers = []
        info_acceptors = []
        cas_redirect_handlers = []
        interceptors = []
        access_control = []
        for plugin in plugins:
            if IResponseContentModifier.providedBy(plugin):
                content_modifiers.append(plugin)
            if IRProxyInfoAcceptor.providedBy(plugin):
                info_acceptors.append(plugin)
            if ICASRedirectHandler.providedBy(plugin):
                cas_redirect_handlers.append(plugin)
            if IResourceInterceptor.providedBy(plugin):
                interceptors.append(plugin)
            if IAccessControl.providedBy(plugin):
                access_control.append(plugin)
        self.info_acceptors = info_acceptors
        content_modifiers.sort(key=lambda x: x.mod_sequence)
        self.content_modifiers = content_modifiers
        cas_redirect_handlers.sort(key=lambda x: x.cas_redirect_sequence)
        self.cas_redirect_handlers = cas_redirect_handlers
        interceptors.sort(key=lambda x: x.interceptor_sequence)
        self.interceptors = interceptors
        access_control.sort(key=lambda x: x.ac_sequence)
        self.access_control = access_control
        # Create static resources.
        static_resources = {}
        for plugin in plugins:
            if IStaticResourceProvider.providedBy(plugin):
                if plugin.static_resource_base in static_resources:
                    if static_resources[
                            plugin.
                            static_resource_base] != plugin.static_resource_dir:
                        raise Exception(
                            "Static resource conflict for '{0}': '{1}' != '{2}'"
                            .format(
                                plugin.static_resource_base,
                                static_resources[plugin.static_resource_base],
                                plugin.static_resource_dir))
                else:
                    static_resources[
                        plugin.
                        static_resource_base] = plugin.static_resource_dir
        self.static_handlers = []
        for n, (resource_base,
                resource_dir) in enumerate(static_resources.iteritems()):
            handler = lambda self, request: File(resource_dir)
            handler = self.app.route(resource_base, branch=True)(handler)
            self.static_handlers.append(handler)

    def log(self, msg, important=False):
        if important or self.verbose:
            if important:
                tag = "INFO"
            else:
                tag = "DEBUG"
            log.msg("[{0}] {1}".format(tag, msg))

    def handle_port_set(self):
        fqdn = self.fqdn
        port = self.port
        proxied_scheme = self.proxied_scheme
        proxied_netloc = self.proxied_netloc
        proxied_path = self.proxied_path
        for plugin in self.info_acceptors:
            plugin.proxy_fqdn = fqdn
            plugin.proxy_port = port
            plugin.proxied_scheme = proxied_scheme
            plugin.proxied_netloc = proxied_netloc
            plugin.proxied_path = proxied_path
            plugin.handle_rproxy_info_set()
            plugin.expire_session = self._expired

    def _make_agents(self, auth_files):
        """
        Configure the web clients that:
        * perform backchannel CAS ticket validation
        * proxy the target site
        """
        self.connectionPool = HTTPConnectionPool(self.reactor)
        if auth_files is None or len(auth_files) == 0:
            agent = Agent(self.reactor, pool=self.connectionPool)
        else:
            extra_ca_certs = []
            for ca_cert in auth_files:
                with open(ca_cert, "rb") as f:
                    data = f.read()
                cert = crypto.load_certificate(crypto.FILETYPE_PEM, data)
                del data
                extra_ca_certs.append(cert)
            policy = CustomPolicyForHTTPS(extra_ca_certs)
            agent = Agent(self.reactor,
                          contextFactory=policy,
                          pool=self.connectionPool)
        if self.proxy_client_endpoint_s is not None:
            self.proxyConnectionPool = HTTPConnectionPool(self.reactor)
            self.proxy_agent = Agent.usingEndpointFactory(
                self.reactor,
                WebClientEndpointFactory(self.reactor,
                                         self.proxy_client_endpoint_s),
                pool=self.proxyConnectionPool)
        else:
            self.proxy_agent = agent
        if self.cas_client_endpoint_s is not None:
            self.casConnectionPool = HTTPConnectionPool(self.reactor)
            self.cas_agent = Agent.usingEndpointFactory(
                self.reactor,
                WebClientEndpointFactory(self.reactor,
                                         self.cas_client_endpoint_s),
                pool=self.casConnectionPool)
        else:
            self.cas_agent = agent

    def is_excluded(self, request):
        resource = request.path
        if resource in self.excluded_resources:
            return True
        for excluded in self.excluded_branches:
            if proxyutils.is_resource_or_child(excluded, resource):
                return True
        return False

    def mod_headers(self, h):
        keymap = {}
        for k, v in h.iteritems():
            key = k.lower()
            if key in keymap:
                keymap[key].append(k)
            else:
                keymap[key] = [k]
        if 'host' in keymap:
            for k in keymap['host']:
                h[k] = [self.proxied_netloc]
        if 'origin' in keymap:
            for k in keymap['origin']:
                h[k] = [self.proxied_netloc]
        if 'content-length' in keymap:
            for k in keymap['content-length']:
                del h[k]
        if 'referer' in keymap:
            referer_handled = False
            keys = keymap['referer']
            if len(keys) == 1:
                k = keys[0]
                values = h[k]
                if len(values) == 1:
                    referer = values[0]
                    new_referer = self.proxy_url_to_proxied_url(referer)
                    if new_referer is not None:
                        h[k] = [new_referer]
                        self.log("Re-wrote Referer header: '%s' => '%s'" %
                                 (referer, new_referer))
                        referer_handled = True
            if not referer_handled:
                for k in keymap['referer']:
                    del h[k]
        return h

    def _check_for_logout(self, request):
        data = request.content.read()
        samlp_ns = "{urn:oasis:names:tc:SAML:2.0:protocol}"
        try:
            root = etree.fromstring(data)
        except Exception as ex:
            root = None
        if (root is not None) and (root.tag == "%sLogoutRequest" % samlp_ns):
            instant = root.get('IssueInstant')
            if instant is not None:
                try:
                    instant = parse_date(instant)
                except ValueError:
                    self.log("Invalid issue_instant supplied: '{0}'.".format(
                        instant),
                             important=True)
                    instant = None
                if instant is not None:
                    utcnow = datetime.datetime.utcnow()
                    seconds = abs(
                        (utcnow -
                         instant.replace(tzinfo=None)).total_seconds())
                    if seconds <= self.logout_instant_skew:
                        results = root.findall("%sSessionIndex" % samlp_ns)
                        if len(results) == 1:
                            result = results[0]
                            ticket = result.text
                            sess_uid = self.logout_tickets.get(ticket, None)
                            if sess_uid is not None:
                                self._expired(sess_uid)
                                return True
                            else:
                                self.log(
                                    ("No matching session for logout request "
                                     "for ticket '{0}'.").format(ticket))
                    else:
                        self.log(("Issue instant was not within"
                                  " {0} seconds of actual time.").format(
                                      self.logout_instant_skew),
                                 important=True)
                else:
                    self.log("Could not parse issue instant.", important=True)
            else:
                self.log("'IssueInstant' attribute missing from root.",
                         important=True)
        elif root is None:
            self.log("Could not parse XML.", important=True)
        return False

    @app.route("/", branch=True)
    def proxy(self, request):
        for pattern in self.logout_patterns:
            if does_url_match_pattern(request.uri, pattern):
                sess = request.getSession()
                sess_uid = sess.uid
                self._expired(sess_uid)
                cas_logout = self.cas_info.get('logout_url', None)
                if cas_logout is not None:
                    if self.logout_passthrough:
                        d = self.reverse_proxy(request, protected=False)
                    return request.redirect(cas_logout)
                else:
                    return self.reverse_proxy(request, protected=False)
        if self.is_excluded(request):
            return self.reverse_proxy(request, protected=False)
        valid_sessions = self.valid_sessions
        sess = request.getSession()
        sess_uid = sess.uid
        if not sess_uid in valid_sessions:
            self.log(("Session {0} not in valid sessions.  "
                      "Will authenticate with CAS.").format(sess_uid))
            if request.method == 'POST':
                headers = request.requestHeaders
                if headers.hasHeader("Content-Type"):
                    ct_list = headers.getRawHeaders("Content-Type")
                    #log.msg("[DEBUG] ct_list: %s" % str(ct_list))
                    for ct in ct_list:
                        if ct.find('text/xml') != -1 or ct.find(
                                'application/xml') != -1:
                            if self._check_for_logout(request):
                                return ""
                            else:
                                break
            # CAS Authentication
            # Does this request have a ticket?  I.e. is it coming back from a successful
            # CAS authentication?
            args = request.args
            ticket_name = self.ticket_name
            if ticket_name in args:
                values = args[ticket_name]
                if len(values) == 1:
                    ticket = values[0]
                    d = self.validate_ticket(ticket, request)
                    return d
            # If no ticket is present, redirect to CAS.
            d = self.redirect_to_cas_login(request)
            return d
        elif request.path == self.auth_info_resource:
            self.log("Providing authentication info.")
            return self.deliver_auth_info(request)
        else:
            d = self.reverse_proxy(request)
            return d

    def deliver_auth_info(self, request):
        valid_sessions = self.valid_sessions
        sess = request.getSession()
        sess_uid = sess.uid
        session_info = valid_sessions[sess_uid]
        username = session_info['username']
        attributes = session_info['attributes']
        doc = {'username': username, 'attributes': attributes}
        serialized = json.dumps(doc)
        request.responseHeaders.setRawHeaders('Content-Type',
                                              ['application/json'])
        return serialized

    def get_url(self, request):
        if self.is_https:
            scheme = 'https'
            default_port = 443
        else:
            scheme = 'http'
            default_port = 80
        fqdn = self.fqdn
        port = self.port
        if port is None:
            port = default_port
        if port == default_port:
            return urlparse.urljoin("%s://%s" % (scheme, fqdn), request.uri)
        else:
            return urlparse.urljoin("%s://%s:%d" % (scheme, fqdn, port),
                                    request.uri)

    def redirect_to_cas_login(self, request):
        """
        Begin the CAS redirection process.
        """
        service_url = self.get_url(request)
        d = None
        for plugin in self.cas_redirect_handlers:
            if d is None:
                d = defer.maybeDeferred(plugin.intercept_service_url,
                                        service_url, request)
            else:
                d.addCallback(plugin.intercept_service_url, request)
        if d is None:
            return self.complete_redirect_to_cas_login(service_url, request)
        else:
            d.addCallback(self.complete_redirect_to_cas_login, request)
            return d

    def complete_redirect_to_cas_login(self, service_url, request):
        """
        Complete the CAS redirection process.
        Return a deferred that will redirect the user-agent to the CAS login.
        """
        cas_info = self.cas_info
        login_url = cas_info['login_url']
        p = urlparse.urlparse(login_url)
        params = {self.service_name: service_url}
        if p.query == '':
            param_str = urlencode(params)
        else:
            qs_map = urlparse.parse_qs(p.query)
            qs_map.update(params)
            param_str = urlencode(qs_map, doseq=True)
        p = urlparse.ParseResult(*tuple(p[:4] + (param_str, ) + p[5:]))
        url = urlparse.urlunparse(p)
        self.log("Redirecting to CAS with URL '{0}'.".format(url))
        d = request.redirect(url)
        return d

    def validate_ticket(self, ticket, request):
        service_name = self.service_name
        ticket_name = self.ticket_name
        this_url = self.get_url(request)
        p = urlparse.urlparse(this_url)
        qs_map = urlparse.parse_qs(p.query)
        if ticket_name in qs_map:
            del qs_map[ticket_name]
        param_str = urlencode(qs_map, doseq=True)
        p = urlparse.ParseResult(*tuple(p[:4] + (param_str, ) + p[5:]))
        service_url = urlparse.urlunparse(p)
        params = {
            service_name: service_url,
            ticket_name: ticket,
        }
        param_str = urlencode(params, doseq=True)
        p = urlparse.urlparse(self.cas_info['service_validate_url'])
        p = urlparse.ParseResult(*tuple(p[:4] + (param_str, ) + p[5:]))
        service_validate_url = urlparse.urlunparse(p)
        self.log("Requesting service-validate URL => '{0}' ...".format(
            service_validate_url))
        http_client = HTTPClient(self.cas_agent)
        d = http_client.get(service_validate_url)
        d.addCallback(treq.content)
        d.addCallback(self.parse_sv_results, service_url, ticket, request)
        return d

    def parse_sv_results(self, payload, service_url, ticket, request):
        self.log("Parsing /serviceValidate results  ...")
        ns = self.ns
        try:
            root = etree.fromstring(payload)
        except (etree.XMLSyntaxError, ) as ex:
            self.log(("error='Error parsing XML payload.' "
                      "service='{0}' ticket={1}'/n{2}").format(
                          service_url, ticket, ex),
                     important=True)
            return self.render_template_500(request)
        if root.tag != ('%sserviceResponse' % ns):
            self.log(
                ("error='Error parsing XML payload.  No `serviceResponse`.' "
                 "service='{0}' ticket={1}'").format(service_url, ticket),
                important=True)
            return self.render_template_403(request)
        results = root.findall("{0}authenticationSuccess".format(ns))
        if len(results) != 1:
            self.log((
                "error='Error parsing XML payload.  No `authenticationSuccess`.' "
                "service='{0}' ticket={1}'").format(service_url, ticket),
                     important=True)
            return self.render_template_403(request)
        success = results[0]
        results = success.findall("{0}user".format(ns))
        if len(results) != 1:
            self.log(
                ("error='Error parsing XML payload.  Not exactly 1 `user`.' "
                 "service='{0}' ticket={1}'").format(service_url, ticket),
                important=True)
            return self.render_template_403(request)
        user = results[0]
        username = user.text
        attributes = success.findall("{0}attributes".format(ns))
        attrib_map = {}
        for attrib_container in attributes:
            for elm in attrib_container.findall('./*'):
                tag_name = elm.tag[len(ns):]
                value = elm.text
                attrib_map.setdefault(tag_name, []).append(value)
        # Access control plugins
        access_control = self.access_control
        for ac_plugin in access_control:
            is_allowed, reason = ac_plugin.isAllowed(username, attrib_map)
            if not is_allowed:
                self.log((
                    "Access denied:  user='******' ac_plugin='{ac_plugin}' "
                    "reason={reason}, service='{service}' ticket='{ticket}'"
                ).format(username=username,
                         ac_plugin=ac_plugin.tagname,
                         service=service_url,
                         ticket=ticket,
                         reason=reason),
                         important=True)
                return self.render_template_403(request,
                                                username=username,
                                                reason=reason)
        # Update session session
        valid_sessions = self.valid_sessions
        logout_tickets = self.logout_tickets
        sess = request.getSession()
        sess_uid = sess.uid
        if sess_uid not in valid_sessions:
            valid_sessions[sess_uid] = {}
        valid_sessions[sess_uid].update({
            'username': username,
            'ticket': ticket,
            'attributes': attrib_map
        })
        if not ticket in logout_tickets:
            logout_tickets[ticket] = sess_uid
        auth_info_callback = self.auth_info_callback
        if auth_info_callback is not None:
            auth_info_callback(username, attrib_map)
        sess.notifyOnExpire(lambda: self._expired(sess_uid))
        # Reverse proxy.
        return request.redirect(service_url)

    def _expired(self, uid):
        valid_sessions = self.valid_sessions
        if uid in valid_sessions:
            session_info = valid_sessions[uid]
            username = session_info['username']
            ticket = session_info['ticket']
            del valid_sessions[uid]
            auth_info_callback = self.auth_info_callback
            if auth_info_callback is not None:
                auth_info_callback(username, None)
            logout_tickets = self.logout_tickets
            if ticket in logout_tickets:
                del logout_tickets[ticket]
            self.log(("label='Expired session.' session_id='{0}' "
                      "username='******'").format(uid, username))

    def reverse_proxy(self, request, protected=True):
        if protected:
            sess = request.getSession()
            valid_sessions = self.valid_sessions
            sess_uid = sess.uid
            username = valid_sessions[sess_uid]['username']
        # Normal reverse proxying.
        kwds = {}
        cookiejar = {}
        kwds['allow_redirects'] = False
        kwds['cookies'] = cookiejar
        req_headers = self.mod_headers(
            dict(request.requestHeaders.getAllRawHeaders()))
        kwds['headers'] = req_headers
        if protected:
            kwds['headers'][self.remoteUserHeader] = [username]
        if request.method in ('PUT', 'POST'):
            kwds['data'] = request.content.read()
        url = self.proxied_url + request.uri
        # Determine if a plugin wants to intercept this URL.
        interceptors = self.interceptors
        for interceptor in interceptors:
            if interceptor.should_resource_be_intercepted(
                    url, request.method, req_headers, request):
                return interceptor.handle_resource(url, request.method,
                                                   req_headers, request)
        # Check if this is a request for a websocket.
        d = self.checkForWebsocketUpgrade(request)
        if d is not None:
            return d
        # Typical reverse proxying.
        self.log("Proxying URL => {0}".format(url))
        http_client = HTTPClient(self.proxy_agent)
        d = http_client.request(request.method, url, **kwds)

        def process_response(response, request):
            req_resp_headers = request.responseHeaders
            resp_code = response.code
            resp_headers = response.headers
            resp_header_map = dict(resp_headers.getAllRawHeaders())
            # Rewrite Location headers for redirects as required.
            if resp_code in (301, 302, 303, 307,
                             308) and "Location" in resp_header_map:
                values = resp_header_map["Location"]
                if len(values) == 1:
                    location = values[0]
                    if request.isSecure():
                        proxy_scheme = 'https'
                    else:
                        proxy_scheme = 'http'
                    new_location = self.proxied_url_to_proxy_url(
                        proxy_scheme, location)
                    if new_location is not None:
                        resp_header_map['Location'] = [new_location]
            request.setResponseCode(response.code, message=response.phrase)
            for k, v in resp_header_map.iteritems():
                if k == 'Set-Cookie':
                    v = self.mod_cookies(v)
                req_resp_headers.setRawHeaders(k, v)
            return response

        def mod_content(body, request):
            """
            Modify response content before returning it to the user agent.
            """
            d = None
            for content_modifier in self.content_modifiers:
                if d is None:
                    d = content_modifier.transform_content(body, request)
                else:
                    d.addCallback(content_modifier.transform_content, request)
            if d is None:
                return body
            else:
                return d

        d.addCallback(process_response, request)
        d.addCallback(treq.content)
        d.addCallback(mod_content, request)
        return d

    def checkForWebsocketUpgrade(self, request):
        def _extract(name):
            raw_value = request.getHeader(name)
            if raw_value is None:
                return set([])
            else:
                return set(raw_value.split(', '))

        upgrade = _extract("Upgrade")
        connection = _extract("Connection")
        if 'websocket' in upgrade and 'Upgrade' in connection:
            uri = request.uri
            proxy_fqdn = self.fqdn
            proxy_port = self.port
            proxied_scheme = self.proxied_scheme
            proxied_netloc = self.proxied_netloc
            proxied_path = self.proxied_path
            if self.is_https:
                scheme = 'wss'
            else:
                scheme = 'ws'
            netloc = "{0}:{1}".format(proxy_fqdn, proxy_port)
            proxy_url = "{0}://{1}{2}".format(scheme, netloc, request.uri)
            if proxied_scheme == 'https':
                proxied_scheme = 'wss'
            else:
                proxied_scheme = 'ws'
            proxied_url = proxyutils.proxy_url_to_proxied_url(
                proxied_scheme,
                proxy_fqdn,
                proxy_port,
                proxied_netloc,
                proxied_path,
                proxy_url,
            )
            origin = proxied_url
            kind = "tcp"
            if proxied_scheme == 'wss':
                kind = 'ssl'
            parts = proxied_netloc.split(":", 1)
            proxied_host = parts[0]
            if len(parts) == 2:
                proxied_port = int(parts[1])
            elif proxied_scheme == 'wss':
                proxied_port = 443
            else:
                proxied_port = 80
            extra = ""  #TODO: SSL options.
            proxied_endpoint_str = "{0}:host={1}:port={2}{3}".format(
                kind, proxied_host, proxied_port, extra)
            if proxied_url is not None:
                resource = makeWebsocketProxyResource(proxy_url,
                                                      proxied_endpoint_str,
                                                      proxied_url,
                                                      request,
                                                      reactor=self.reactor,
                                                      origin=origin,
                                                      verbose=self.verbose)
                return resource
        return None

    def mod_cookies(self, value_list):
        proxied_path = self.proxied_path
        proxied_path_size = len(proxied_path)
        results = []
        for cookie_value in value_list:
            c = Cookie.SimpleCookie()
            c.load(cookie_value)
            for k in c.keys():
                m = c[k]
                if m.has_key('path'):
                    m_path = m['path']
                    if self.is_proxy_path_or_child(m_path):
                        m_path = m_path[proxied_path_size:]
                        m['path'] = m_path
            results.append(c.output(header='')[1:])
        return results

    def is_proxy_path_or_child(self, path):
        return proxyutils.is_proxy_path_or_child(self.proxied_path, path)

    def proxied_url_to_proxy_url(self, proxy_scheme, target_url):
        return proxyutils.proxied_url_to_proxy_url(proxy_scheme, self.fqdn,
                                                   self.port,
                                                   self.proxied_netloc,
                                                   self.proxied_path,
                                                   target_url)

    def proxy_url_to_proxied_url(self, target_url):
        return proxyutils.proxy_url_to_proxied_url(self.proxied_scheme,
                                                   self.fqdn, self.port,
                                                   self.proxied_netloc,
                                                   self.proxied_path,
                                                   target_url)

    def get_template_static_base(self):
        if self.template_resource is None:
            return None
        else:
            return '{0}static/'.format(self.template_resource)

    def render_template_403(self, request, **kwargs):
        template_dir = self.template_dir
        if template_dir is None:
            request.setResponseCode(403)
            return ""
        else:
            return self.render_template('error/403.jinja2',
                                        request=request,
                                        **kwargs)

    def render_template_500(self, request, **kwargs):
        template_dir = self.template_dir
        if template_dir is None:
            request.setResponseCode(500)
            return ""
        else:
            return self.render_template('error/500.jinja2',
                                        request=request,
                                        **kwargs)

    def render_template(self, template_name, **kwargs):
        template_dir = self.template_dir
        try:
            template = self.template_loader_.load(self.template_env_,
                                                  template_name)
        except TemplateNotFound:
            raise Exception(
                "The template '{0}' was not found.".format(template_name))
        return template.render(static_base=self.static_base,
                               **kwargs).encode('utf-8')

    def create_template_static_resource(self):
        static_path = os.path.join(self.template_dir, 'static')
        static = File(static_path)
        return static

    def static(self, request):
        return self.templateStaticResource_

    @app.handle_errors(Exception)
    def handle_uncaught_errors(self, request, failure):
        self.log("Uncaught exception: {0}".format(failure), important=True)
        return self.render_template_500(request=request, failure=failure)
Beispiel #38
0
# #!/usr/bin/python3

from klein import run, route, Klein
import json
import os
import pickle

app = Klein()

# nlp = spacy.load('en_core_web_sm')
# def tokeni(sentence):
#     return list([str(x) for x in nlp.tokenizer(sentence)])

def load_model(filename):
    with open(filename, 'rb') as f:
        model_struct = pickle.load(f)
        clf = model_struct['model']
    return clf

def predict_class_from_text(clf,text):
    predicted = clf.predict([text])[0]
    return predicted

def predict_proba_from_text(clf,text):
    proba = clf.predict_proba([text])[0][1]
    return int(proba*100)

@app.route("/ecommerce-category", methods=['POST'])
def get_category(request):
    content = json.loads(request.content.read())
    text = content['content']
Beispiel #39
0
from klein import Klein
from twisted.internet import task, reactor
from twisted.web.resource import Resource

import Sequential

app = Klein()

def delayedCall(n):
    return 'Delayed for %d seconds' % n

@app.route('/delay/<int:t>')
def delay(requests, t):
    """
    Delay the response by t seconds
    """
    if t > 0:
        d = task.deferLater(reactor, t, delayedCall, t)
        return d
    else:
        return delayedCall(0)

@app.route('/upload', branch=True)
def sequential(request):
    return Sequential.app.resource()


app.run('localhost', 8000)
Beispiel #40
0
class Api:  # noqa: D101
	app = Klein()

	default = {  # note that these defaults are documented in configuration/www.rst
		"left_pad": 5,
		"left_text": "Build Status",
		"left_color": "#555",
		"right_pad": 5,
		"border_radius": 5,
		"style": "plastic",
		"template_name": "{style}.svg.j2",
		"font_face": "DejaVu Sans",
		"font_size": 11,
		"color_scheme": color_scheme,
	}

	def __init__(self, ep):
		self.ep = ep
		self.env = jinja2.Environment(
				loader=jinja2.
				ChoiceLoader([jinja2.PackageLoader("buildbot_badges"), jinja2.FileSystemLoader("templates")])
				)

	def makeConfiguration(self, request):  # noqa: D102

		config = {}
		config.update(self.default)
		for k, v in self.ep.config.items():
			if k == "color_scheme":
				config[k].update(v)  # type: ignore
			else:
				config[k] = v

		for k, v in request.args.items():
			k = bytes2unicode(k)
			config[k] = escape(bytes2unicode(v[0]))
		return config

	@app.route("/<path:builder>.png", methods=["GET"])
	@defer.inlineCallbacks
	def getPng(self, request, builder):  # noqa: D102
		svg = yield self.getSvg(request, builder)
		request.setHeader("content-type", "image/png")
		defer.returnValue(cairosvg.svg2png(svg))

	@app.route("/<path:builder>.svg", methods=["GET"])
	@defer.inlineCallbacks
	def getSvg(self, request, builder):  # noqa: D102

		master: buildbot.master.BuildMaster = self.ep.master

		if not builder.isidentifier():
			builder = yield master.db.builders.findBuilderId(builder, False)

		config = self.makeConfiguration(request)
		request.setHeader("content-type", "image/svg+xml")
		request.setHeader("cache-control", "no-cache")

		# get the last build for that builder using the data api
		last_build = yield self.ep.master.data.get(("builders", builder, "builds"), limit=1, order=["-number"])

		# get the status text corresponding to results code
		results_txt = "unknown"
		if last_build:
			results = last_build[0]["results"]
			complete = last_build[0]["complete"]
			if not complete:
				results_txt = "running"
			elif results >= 0 and results < len(Results):
				results_txt = Results[results]

		svgdata = self.makesvg(results_txt, results_txt, left_text=config["left_text"], config=config)
		defer.returnValue(svgdata)

	def textwidth(self, text, config):
		"""
		Calculates the width of the specified text.
		"""

		surface = cairo.SVGSurface(None, 1280, 200)
		ctx = cairo.Context(surface)
		ctx.select_font_face(config["font_face"], cairo.FONT_SLANT_NORMAL, cairo.FONT_WEIGHT_NORMAL)
		ctx.set_font_size(int(config["font_size"]))
		return ctx.text_extents(text)[4]

	def makesvg(self, right_text, status=None, left_text=None, left_color=None, config=None):
		"""
		Renders an SVG from the template, using the specified data.
		"""

		right_color = config["color_scheme"].get(status, "#9f9f9f")  # Grey

		left_text = left_text or config["left_text"]
		left_color = left_color or config["left_color"]

		left = {"color": left_color, "text": left_text, "width": self.textwidth(left_text, config)}
		right = {"color": right_color, "text": right_text, "width": self.textwidth(right_text, config)}

		template = self.env.get_template(config["template_name"].format(**config))
		return template.render(left=left, right=right, config=config)
class orderResource(object):
    app = Klein()

    def __init__(self, dbPools, mqPools):
        self.dbPools = dbPools
        self.mqPools = mqPools

    @app.route('/<path:catchall>')
    def catchAll(self, request, catchall):
        request.setHeader('Content-Type', 'application/json')
        request.setResponseCode(404)
        return responseJson(
            API_RESPONSE_CODE_ENDPOINT_NOT_EXISTS,
            API_RESPONSE_MSG.get(API_RESPONSE_CODE_ENDPOINT_NOT_EXISTS))

    @app.route('/')
    def index(self, request):
        request.setHeader('Content-Type', 'application/json')
        request.setResponseCode(404)
        return responseJson(API_RESPONSE_CODE_SUCCESS,
                            API_RESPONSE_MSG.get(API_RESPONSE_CODE_SUCCESS))

    @app.route('/order/create/<string:name>', methods=['POST'])
    def create(self, request, name):
        def createOrderSuccess(result, tradeNo):
            if False != result:
                log.msg('[apiService, %s] %s: order create success' % (
                    name,
                    tradeNo,
                ))
                publishData = {'agentUid': result[0], 'tradeNo': result[1]}
                self.publishOrderData(name, tradeNo, dictToJson(publishData))
            else:
                log.msg('[apiService, %s] %s: order create success' % (
                    name,
                    tradeNo,
                ))
                log.msg('[apiService, %s] %s: result:' % (
                    name,
                    tradeNo,
                ), result)

        def createOrderFail(error, tradeNo):
            log.msg('[apiService, %s] %s: order create fail' % (
                name,
                tradeNo,
            ))
            log.msg('[apiService, %s] %s: error:' % (
                name,
                tradeNo,
            ), error)

        def createOrder(txn, productId, agentUid, tradeNo):
            ret = False
            tradeGoodsTbName = getAppConfig('%s.table.trade_goods' % (name, ))
            tradeOrderTbName = getAppConfig('%s.table.trade_order' % (name, ))
            agentUserTbName = getAppConfig('%s.table.agent_user' % (name, ))
            qUserTbName = getAppConfig('%s.table.q_user' % (name, ))
            txn.execute(
                'SELECT name,price,amount,bonus,gameid FROM ' +
                tradeGoodsTbName +
                ' WHERE autoid = %s AND status = 1 AND level = 2',
                (productId, ))
            goodsData = txn.fetchone()
            if goodsData is not None:
                txn.execute(
                    'SELECT u.uid, u.username, m.uid AS pagentid, m.code AS pcode FROM '
                    + agentUserTbName + ' AS m INNER JOIN ' + qUserTbName +
                    ' AS u ON u.agentid = m.uid WHERE m.uid = %s',
                    (agentUid, ))
                userData = txn.fetchone()
                if userData is not None:
                    txn.execute(
                        'INSERT INTO ' + tradeOrderTbName +
                        '(`uid`, `appuname`, `goodid`, `tradeno`, `tradetype`, `coin`, `pagentid`, `pcode`, `amount`) VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s)',
                        (userData.get('uid'), userData.get('username'),
                         productId, tradeNo,
                         'AGENTPAY', goodsData.get('price'),
                         userData.get('pagentid'), userData.get('pcode'),
                         goodsData.get('amount') + goodsData.get('bonus')))
                    ret = (agentUid, tradeNo)
            return ret

        tradeNo = "%s%s%s" % (
            10,
            getGameIdBySymbol(name),
            getTradeNo(),
        )
        log.msg('[apiService, %s] %s: order create parms' % (
            name,
            tradeNo,
        ), request.args)
        parmsRule = {
            'agent_uid': {
                'name': requestValidation.RULE_NAME_REQUIRED,
                'error': API_RESPONSE_CODE_INVALID_AGENT_UID
            },
            'good_id': {
                'name': requestValidation.RULE_NAME_REQUIRED,
                'error': API_RESPONSE_CODE_INVALID_PRODUCTID
            },
            'timestamp': {
                'name': requestValidation.RULE_NAME_NUMBER,
                'error': API_RESPONSE_CODE_INVALID_TIMESTAMP
            },
            'sign': {
                'name': requestValidation.RULE_NAME_REQUIRED,
                'error': API_RESPONSE_CODE_INVALID_SIGN
            }
        }
        gameId = getGameIdBySymbol(name)
        if gameId is None:
            ret = API_RESPONSE_CODE_INVALID_GAMEID
        else:
            _requestValidation = requestValidation(request, parmsRule)
            if True == _requestValidation.validation():
                parmsSign = request.args.get('sign')[0]
                parms = request.args
                parms.pop('sign')
                if True != identifySign(parms, parmsSign):
                    ret = API_RESPONSE_CODE_INVALID_SIGN
                else:
                    productId = request.args.get('good_id')[0]
                    agentUid = request.args.get('agent_uid')[0]
                    log.msg(self.dbPools)
                    dbDefer = self.dbPools[name].runInteraction(
                        createOrder, productId, agentUid, tradeNo)
                    dbDefer.addCallback(createOrderSuccess, tradeNo)
                    dbDefer.addErrback(createOrderFail, tradeNo)
                    ret = API_RESPONSE_CODE_SUCCESS
            else:
                ret = _requestValidation.getErrorCode()

        request.setHeader('Content-Type', 'application/json')
        return responseJson(ret, API_RESPONSE_MSG.get(ret))

    @app.route('/order/info/<string:name>')
    def info(self, request, name):
        request.setHeader('Content-Type', 'application/json')
        return responseJson(API_RESPONSE_CODE_SUCCESS,
                            API_RESPONSE_MSG.get(API_RESPONSE_CODE_SUCCESS))

    def publishOrderData(self, game, tradeNo, data):
        try:
            mqConn = self.mqPools[game]
            mqChannel = mqConn.channel()
            queueConfig = getMqQueueConfig(game)
            exchange = queueConfig.get('exchange')
            queueName = queueConfig.get('queueName')
            routingKey = queueConfig.get('routingKey')
            mqChannel.confirm_delivery()
            mqChannel.exchange_declare(exchange=exchange, durable=True)
            mqChannel.queue_declare(queue=queueName, durable=True)
            mqChannel.queue_bind(queue=queueName,
                                 exchange=exchange,
                                 routing_key=routingKey)
            mqChannel.publish(exchange=exchange,
                              routing_key=routingKey,
                              body=data)
            log.msg(
                '[apiService, %s] %s: order publish success:' % (
                    game,
                    tradeNo,
                ), data)
        except Exception, e:
            log.msg(
                '[apiService, %s] %s: order publish fail: %s' % (
                    game,
                    tradeNo,
                ), data, e.message)
            log.msg(e)
        finally:
Beispiel #42
0
 def setUp(self):
     self.app = Klein()
     self.kr = KleinResource(self.app)
Beispiel #43
0
 class _One(object):
     oneKlein = Klein()
     @oneKlein.route("/foo")
     def foo(self):
         pass

# This is our WAMP application
##
wampapp = Application(u'com.example')


@wampapp.register()
def square(x):
    print("square() called with {}".format(x))
    return x * x


# This is our Web application
##
webapp = Klein()
webapp.visits = 0
webapp.templates = jinja2.Environment(loader=jinja2.FileSystemLoader('templates'))


@webapp.route('/')
def home(request):
    webapp.visits += 1
    wampapp.session.publish(u'com.example.onvisit', visits=webapp.visits)
    page = webapp.templates.get_template('index.html')
    return page.render(visits=webapp.visits)


@webapp.route('/square/<int:x>')
@inlineCallbacks
def square(request, x):
Beispiel #45
0
import treq
from klein import Klein
from twisted.internet.defer import inlineCallbacks, returnValue
from bs4 import BeautifulSoup
import json
import re
from twisted.python import log
app = Klein()

comp_name_regex = re.compile(r'\b[\w ]+\b')


def clean_text(str,flag='td'):
	if flag == 'td':
		return str.find('td').get_text().replace('-',' ').replace(',','').replace('.','').strip().encode('utf-8','ignore')
	elif flag == 'th':
		return str.find('th').get_text().strip().replace('-',' ').replace(',','').replace('.','').split(':')[0].encode('utf-8','ignore')	


@app.route('/')
def helloworld():
	return "<h>Helloe!!<\h>"

@app.route('/apis/get',methods=['GET',])
@inlineCallbacks
def check_status(request):
	if not request.args.get("cname"):
		pass
		# abort(400)
	# elif not re.match(request.args.get("cname")):	
	else:
# coding=utf-8
import os

from klein import Klein
from twisted.internet import reactor
from twisted.web.resource import Resource, ForbiddenResource
from twisted.web.server import Site
from twisted.web.static import File

from server.app import App

class FileNoDir(File):
    def directoryListing(self):
        return ForbiddenResource()

if __name__ == '__main__':
    client_side_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'client_side'))
    server = Klein()
    app = App(server, client_side_dir)
    root = server.resource()
    site = Site(root)
    """
    root.putChild("static", FileNoDir(os.path.join(client_side_dir, 'static')))
    root.putChild("app", FileNoDir(os.path.join(client_side_dir, 'app')))
    root.putChild("", File(os.path.join(client_side_dir, 'templates', 'index.html')))
    root.putChild("templates", File(os.path.join(client_side_dir, 'templates')))
    """
    reactor.listenTCP(8888, site)
    reactor.run()
class ItemStore(object):
    app = Klein()

    def _init_(self):
        self._items = {}

    @app.route('/')
    def items(self, request):
        request.setHeader('Content-Type', 'application/json')
        return json.dumps(self._items)

    @app.route('/<string:name>', methods=['POST'])
    def post_item(self, request, name):
        request.setHeader('Content-Type', 'application/json')
        content = json.loads(request.content.read())

        print("123langCode")
        print(content["langCode"])
        langCode = content["langCode"]
        text = content["data"]

        print("pure text")
        print(text)

        text = text\
            .replace("\r", " ")\
            .replace("\n", " ") \
            .replace("gm", " ") \
            .replace("mg/dl", " ")\
            .replace("high", " ") \
            .lower()

        text = re.sub(r'[^A-Za-z ]+', " is a ", text)
        text = re.sub(' +', " ", text)

        text = text \
            .replace("high", " ")\
            .replace("desirable", " ")\
            .replace("borderline", " ")

        text = re.sub(' +', " ", text)

        print("regexed text")
        print(text)

        sentence = sp(text)
        text = " ".join([token.lemma_ for token in sentence])

        print("lemmatized text")
        print(text)

        doc = nlp(json.dumps(text))
        entities = doc.ents

        entitySet = set()

        for entity in entities:
            entitySet.add(str(entity))
            splits = str(entity).split(" ")

            for split in splits:
                entitySet.add(str(split).strip())

        print(entitySet)

        return self.getQueryResults(entitySet, langCode)

    def getQueryResults(self, entitySet, langCode):
        data = {'entities': []}

        for entity in entitySet:
            # result_entity = dict()
            result_entity = {"entityName": "", "comment": "", "foods": [], "diseases": []}

            abstract_sparql_query = self.getAbstractQuery(entity, langCode)
            abstract_query_results = abstract_sparql_query.query().convert()

            print("abstract_query_results")
            print(abstract_query_results)

            if len(abstract_query_results["results"]["bindings"]) == 0:
                continue

            for result in abstract_query_results["results"]["bindings"]:
                result_entity["entityName"] = entity
                #result_entity['entityURI'] = str(result["chem"]["value"])
                result_entity["comment"] = str(result["comment"]["value"])
                #print(result_entity)

            # food_sparql_query = self.getFoodQuery(entity, langCode)
            # food_query_results = food_sparql_query.query().convert()
            #
            # for result in food_query_results["results"]["bindings"]:
            #     result_entity["foods"].append(str(result["label"]["value"]))

            # food_dict = dict()
            # food_sparql_query = self.getLowFoodQuery(entity, langCode)
            # food_query_results = food_sparql_query.query().convert()
            #
            # for result in food_query_results["results"]["bindings"]:
            #     result_entity["foods"].append(str(result["label"]["value"]))

            food_sparql_query = self.getFoodQuery(entity, langCode)
            food_query_results = food_sparql_query.query().convert()

            for result in food_query_results["results"]["bindings"]:
                result_entity["foods"].append(str(result["label"]["value"]))


            disease_sparql_query = self.getDiseaseQuery(entity, langCode)
            disease_query_results = disease_sparql_query.query().convert()

            for result in disease_query_results["results"]["bindings"]:
                result_entity["diseases"].append(str(result["label"]["value"]))

            data['entities'].append(result_entity)

        return json.dumps(data)

    def getAbstractQuery(self, entity, langCode):
        sparql = SPARQLWrapper("http://dbpedia.org/sparql")
        sparql.setReturnFormat(JSON)
        query = """
            PREFIX dbo: <http://dbpedia.org/ontology/>
        PREFIX dbp: <http://dbpedia.org/resource/>
        PREFIX foaf: <http://xmlns.com/foaf/0.1/>
        PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
        PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>

        SELECT distinct(?chem) ?comment where 
           { 
            ?chem rdf:type dbo:ChemicalCompound .
            ?chem rdfs:label ?label .
            ?chem rdfs:comment ?comment .
            FILTER regex(?label, "^%s$", "i")
            FILTER (langMatches(lang(?comment),"%s"))
           }
        """ % (entity, langCode)

        #print("query")
        #print(query)

        sparql.setQuery(query)

        return sparql

    def getFoodQuery(self, entity, langCode):
        sparql = SPARQLWrapper("http://dbpedia.org/sparql")
        sparql.setReturnFormat(JSON)
        query = """
            PREFIX dbo: <http://dbpedia.org/ontology/>
        PREFIX dbp: <http://dbpedia.org/resource/>
        PREFIX foaf: <http://xmlns.com/foaf/0.1/>
        PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
        PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>

        SELECT distinct(?food) ?label where 
           { 
            ?food rdf:type dbo:Food .
            ?food rdfs:label ?label .
            ?food dbo:abstract ?abstract .
            FILTER regex(?abstract, "%s", "i")
            FILTER (langMatches(lang(?label),"%s"))
            FILTER (langMatches(lang(?abstract),"%s"))
           }
        """ % (entity, langCode, langCode)

        sparql.setQuery(query)

        return sparql

    def getLowFoodQuery(self, entity, langCode):
        sparql = SPARQLWrapper("http://dbpedia.org/sparql")
        sparql.setReturnFormat(JSON)
        query = """
            PREFIX dbo: <http://dbpedia.org/ontology/>
        PREFIX dbp: <http://dbpedia.org/resource/>
        PREFIX foaf: <http://xmlns.com/foaf/0.1/>
        PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
        PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>

        SELECT distinct(?food) ?label where 
           { 
            ?food rdf:type dbo:Food .
            ?food rdfs:label ?label .
            ?food dbo:abstract ?abstract .
            FILTER regex(?abstract, "%s", "i")
            FILTER (langMatches(lang(?label),"%s"))
            FILTER (langMatches(lang(?abstract),"%s"))
           }
        """ % (entity, langCode, langCode)

        sparql.setQuery(query)

        return sparql


    def getDiseaseQuery(self, entity, langCode):
        sparql = SPARQLWrapper("http://dbpedia.org/sparql")
        sparql.setReturnFormat(JSON)
        query = """
        PREFIX dbo: <http://dbpedia.org/ontology/>
        PREFIX dbp: <http://dbpedia.org/resource/>
        PREFIX foaf: <http://xmlns.com/foaf/0.1/>
        PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
        PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>

        SELECT distinct(?chem) ?label where 
           { 
            ?chem rdf:type dbo:Disease .
            ?chem rdfs:label ?label .
            ?chem dbo:abstract ?abstract .
            FILTER regex(?abstract, "%s", "i")
            FILTER (langMatches(lang(?label),"%s"))
            FILTER (langMatches(lang(?abstract),"%s"))
           }
        """ % (entity, langCode, langCode)

        sparql.setQuery(query)

        return sparql
Beispiel #48
0
# -*- coding: utf-8 -*-
import mimetypes
import gc

from klein import Klein
from twisted.web.static import File

from basement import settings
from basement.utils import render_template
from service import service_registry

app = Klein()


@app.route("/")
def index(request):
    return render_template('index.html')


@app.route("/bits/static/", branch=True)
def static(request):
    return File(settings.STATIC_DIR)


@app.route('/<string:service>/<string:package>/<string:markwhat>/')
def badge_markwhat(request, service, package, markwhat):
    gc.collect()
    markwhat = markwhat.lower()

    if markwhat not in settings.MARKWAHT_TEMPLATES:
        request.setResponseCode(400)
klein_app = Klein()

import ConfigParser
from twisted.python import log
import sys
from leap.common.events import server as events_server
from pixelated.config import app_factory
import pixelated.config.args as input_args
import pixelated.bitmask_libraries.register as leap_register
from pixelated.bitmask_libraries.leap_srp import LeapAuthException
import pixelated.config.credentials_prompt as credentials_prompt
import pixelated.support.ext_protobuf  # monkey patch for protobuf in OSX
import pixelated.support.ext_sqlcipher  # monkey patch for sqlcipher in debian


app = Klein()
app.config = {}


def setup():
    args = input_args.parse()
    setup_debugger(args.debug)

    if args.register:
        register(*args.register[::-1])
    else:
        if args.dispatcher:
            config = fetch_credentials_from_dispatcher(args.dispatcher)
            app.config['LEAP_SERVER_NAME'] = config['leap_provider_hostname']
            app.config['LEAP_USERNAME'] = config['user']
            app.config['LEAP_PASSWORD'] = config['password']
Beispiel #50
0
import os

from klein import Klein
from twisted.web import server, static
from twisted.internet import reactor

app = Klein()
# staticFileDir = os.environ['HOME']
staticFileDir = './static'


@app.route('/static/', branch=True)
def staticFiles(request):
    return static.File(staticFileDir)


app.run('localhost', 9000)
Beispiel #51
0
from klein import Klein
app = Klein()

@app.route('/user/<username>')
def pg_user(request, username):
    return 'Hi %s!' % (username,)

@app.route('/user/bob')
def pg_user_bob(request):
    return 'Hello there bob!'

app.run("localhost", 8080)
Beispiel #52
0
class BadlyBehavedHeaders(Headers):
    """
    Make L{Headers} lie, and refuse to return a Host header from
    getAllRequestHeaders.
    """
    def getAllRawHeaders(self):
        # type: () -> Iterable[Tuple[bytes, List[bytes]]]
        """
        Don't return a host header.
        """
        for key, values in super(BadlyBehavedHeaders, self).getAllRawHeaders():
            if key != b'Host':
                yield (key, values)


router = Klein()
requirer = Requirer()


@requirer.require(router.route("/hello/world", methods=['GET']),
                  url=RequestURL())
def requiresURL(url):
    # type: (DecodedURL) -> Text
    """
    This is a route that requires a URL.
    """
    return url.child(u"hello/ world").asText()


class ISample(Interface):
    """
Beispiel #53
0
import treq
from klein import Klein
from twisted.internet.defer import inlineCallbacks, returnValue
from bs4 import BeautifulSoup
import json
import re
# from flask import jsonify

app = Klein()

comp_name_regex = re.compile(r'\b[\w ]+\b')


def clean_text(str, flag='td'):
    if flag == 'td':
        return str.find('td').get_text().replace('-', ' ').replace(
            ',', '').replace('.', '').strip().encode('utf-8', 'ignore')
    elif flag == 'th':
        return str.find('th').get_text().strip().replace('-', ' ').replace(
            ',', '').replace('.', '').split(':')[0].encode('utf-8', 'ignore')


@app.route('/')
def helloworld():
    return "<h>Helloe!!<\h>"


@app.route('/apis/get', methods=[
    'GET',
])
@inlineCallbacks
Beispiel #54
0
class RasaNLU(object):
    """Class representing Rasa NLU http server"""

    app = Klein()

    def __init__(self, config, component_builder=None, testing=False):
        logging.basicConfig(filename=config['log_file'],
                            level=config['log_level'])
        logging.captureWarnings(True)
        logger.debug("Configuration: " + config.view())

        logger.debug("Creating a new data router")
        self.config = config
        self.data_router = self._create_data_router(config, component_builder)
        self._testing = testing
        reactor.suggestThreadPoolSize(config['num_threads'] * 5)

    def _create_data_router(self, config, component_builder):
        return DataRouter(config, component_builder)

    @app.route("/", methods=['GET', 'OPTIONS'])
    @check_cors
    def hello(self, request):
        """Main Rasa route to check if the server is online"""
        return "hello from Rasa NLU: " + __version__

    @app.route("/parse", methods=['GET', 'POST', 'OPTIONS'])
    @requires_auth
    @check_cors
    @inlineCallbacks
    def parse_get(self, request):
        request.setHeader('Content-Type', 'application/json')
        if request.method.decode('utf-8', 'strict') == 'GET':
            request_params = {key.decode('utf-8', 'strict'): value[0].decode('utf-8', 'strict')
                              for key, value in request.args.items()}
        else:
            request_params = simplejson.loads(
                    request.content.read().decode('utf-8', 'strict'))

        if 'query' in request_params:
            request_params['q'] = request_params.pop('query')

        if 'q' not in request_params:
            request.setResponseCode(404)
            dumped = json_to_string({"error": "Invalid parse parameter specified"})
            returnValue(dumped)
        else:
            data = self.data_router.extract(request_params)
            try:
                request.setResponseCode(200)
                response = yield (self.data_router.parse(data) if self._testing
                                  else threads.deferToThread(self.data_router.parse, data))
                returnValue(json_to_string(response))
            except InvalidProjectError as e:
                request.setResponseCode(404)
                returnValue(json_to_string({"error": "{}".format(e)}))
            except Exception as e:
                request.setResponseCode(500)
                logger.exception(e)
                returnValue(json_to_string({"error": "{}".format(e)}))

    @app.route("/version", methods=['GET', 'OPTIONS'])
    @requires_auth
    @check_cors
    def version(self, request):
        """Returns the Rasa server's version"""

        request.setHeader('Content-Type', 'application/json')
        return json_to_string({'version': __version__})

    @app.route("/config", methods=['GET', 'OPTIONS'])
    @requires_auth
    @check_cors
    def rasaconfig(self, request):
        """Returns the in-memory configuration of the Rasa server"""

        request.setHeader('Content-Type', 'application/json')
        return json_to_string(self.config.as_dict())

    @app.route("/status", methods=['GET', 'OPTIONS'])
    @requires_auth
    @check_cors
    def status(self, request):
        request.setHeader('Content-Type', 'application/json')
        return json_to_string(self.data_router.get_status())

    @app.route("/train", methods=['POST', 'OPTIONS'])
    @requires_auth
    @check_cors
    @inlineCallbacks
    def train(self, request):
        data_string = request.content.read().decode('utf-8', 'strict')
        kwargs = {key.decode('utf-8', 'strict'): value[0].decode('utf-8', 'strict')
                  for key, value in request.args.items()}
        request.setHeader('Content-Type', 'application/json')

        try:
            request.setResponseCode(200)
            response = yield self.data_router.start_train_process(
                    data_string, kwargs)
            returnValue(json_to_string({'info': 'new model trained: {}'.format(response)}))
        except AlreadyTrainingError as e:
            request.setResponseCode(403)
            returnValue(json_to_string({"error": "{}".format(e)}))
        except InvalidProjectError as e:
            request.setResponseCode(404)
            returnValue(json_to_string({"error": "{}".format(e)}))
        except TrainingException as e:
            request.setResponseCode(500)
            returnValue(json_to_string({"error": "{}".format(e)}))

    @app.route("/evaluate", methods=['POST'])
    @requires_auth
    @check_cors
    def evaluate(self, request):
        data_string = request.content.read().decode('utf-8', 'strict')
        params = {
            key.decode('utf-8', 'strict'): value[0].decode('utf-8', 'strict')
            for key, value in request.args.items()
        }

        request.setHeader('Content-Type', 'application/json')

        try:
            request.setResponseCode(200)
            response = self.data_router.evaluate(data_string,
                                                 params.get('project'),
                                                 params.get('model'))
            return simplejson.dumps(response)
        except Exception as e:
            request.setResponseCode(500)
            return simplejson.dumps({"error": "{}".format(e)})
Beispiel #55
0
# -*- coding: utf-8 -*-
from __future__ import print_function, unicode_literals
# Cobble together a deterministic random function using a string as a seed.
from random import Random
from hashlib import sha256
from struct import unpack

def random_from_string(string):
    return Random(
        unpack("!I", sha256(string.encode("utf-8")).digest()[:4])[0]
    )

from twisted.web.template import tags, slot
from klein import Klein, Plating
app = Klein()

myStyle = Plating(
    tags=tags.html(
        tags.head(tags.title(slot("pageTitle"))),
        tags.body(tags.h1(slot("pageTitle"), Class="titleHeading"),
                  tags.div(slot(Plating.CONTENT)))
    ),
    defaults={"pageTitle": "Places & Foods"}
)

@myStyle.routed(
    app.route("/"),
    tags.div(
        tags.h2("Sample Places:"),
        tags.ul([tags.li(tags.a(href=["/places/", place])(place))
                 for place in ["new york", "san francisco", "shanghai"]]),
Beispiel #56
0
import pickle
from klein import Klein
import json
from twisted.internet import defer
app = Klein()

class Order(object):
	def __getstate__(self):
		return self.json_dict()

	def __init__(self, order_id):
		self.order_id = order_id
		self.mid = None
		self.tid = None
		self.amount = None
		self.payment = None

	def json_dict(self):
		return {
			'order_id': self.order_id,
			'mid': self.mid,
			'tid': self.tid,
			'amount': self.amount,
			'payment': self.payment
		}

class Orders(object):

	def __init__(self):
		try:
			f = open('/tmp/orders.pickle', 'rb')
class WebInterface(YomboLibrary, BuildDistribution, ErrorHandler, Render,
                   WebServer):
    """
    Web interface framework.
    """
    webapp = Klein()  # Like Flask, but for twisted

    visits = 0
    starting = True
    already_starting_web_servers = False
    hook_listeners = {}  # special way to toss hook calls to routes.

    def _init_(self, **kwargs):
        self.frontend_building = False
        self.web_interface_fully_started = False
        self.enabled = self._Configs.get("webinterface", "enabled", True)

        self.fqdn = self._Configs.get2("dns", "fqdn", None, False)

        self.enabled = self._Configs.get("core", "enabled", True)
        if not self.enabled:
            return

        self.file_cache = ExpiringDict(
            max_len=100, max_age_seconds=120
        )  # used to load a few static files into memory that are commonly used.
        self.translators = {}
        self.idempotence = self._Cache.ttl(name="lib.webinterface.idempotence",
                                           ttl=300)

        self.working_dir = self._Atoms.get("working_dir")
        self.app_dir = self._Atoms.get("app_dir")
        self.wi_dir = "/lib/webinterface"

        self.misc_wi_data = {}

        self.wi_port_nonsecure = self._Configs.get2("webinterface",
                                                    "nonsecure_port", 8080)
        self.wi_port_secure = self._Configs.get2("webinterface", "secure_port",
                                                 8443)

        self.webapp.templates = jinja2.Environment(
            loader=jinja2.FileSystemLoader(f"{self.app_dir}/yombo"),
            extensions=["jinja2.ext.loopcontrols"])
        self.setup_basic_filters()

        self.web_interface_listener = None
        self.web_interface_ssl_listener = None

        self.api_stream_spectators = {}

        # Load API routes
        route_api_v1_atoms(self.webapp)
        route_api_v1_automation_rules(self.webapp)
        # route_api_v1_camera(self.webapp)
        route_api_v1_debug(self.webapp)
        route_api_v1_device(self.webapp)
        route_api_v1_device_command(self.webapp)
        # route_api_v1_events(self.webapp)
        # route_api_v1_gateway(self.webapp)
        # route_api_v1_module(self.webapp)
        route_api_v1_mqtt(self.webapp)
        # route_api_v1_notification(self.webapp)
        route_api_v1_scenes(self.webapp)
        # route_api_v1_server(self.webapp)
        # route_api_v1_statistics(self.webapp)
        # route_api_v1_stream(self.webapp, self)
        route_api_v1_states(self.webapp)
        route_api_v1_system(self.webapp)
        # route_api_v1_storage(self.webapp)
        route_api_v1_user(self.webapp)
        # route_api_v1_webinterface_logs(self.webapp)

        # Load web server routes
        route_home(self.webapp)
        route_misc(self.webapp)
        route_system(self.webapp)
        route_user(self.webapp)
        if self.operating_mode != "run":
            from yombo.lib.webinterface.routes.restore import route_restore
            from yombo.lib.webinterface.routes.setup_wizard import route_setup_wizard
            route_setup_wizard(self.webapp)
            route_restore(self.webapp)

        self.npm_build_results = None

        self.temp_data = ExpiringDict(max_age_seconds=1800)
        self.web_server_started = False
        self.web_server_ssl_started = False

        self.web_factory = None
        self.user_login_tokens = self._Cache.ttl(name="lib.users.cache",
                                                 ttl=300)

    @property
    def operating_mode(self):
        return self._Loader.operating_mode

    def _load_(self, **kwargs):
        if not self.enabled:
            return

        if self.operating_mode == "run":
            self.build_dist()  # Make all the JS and CSS files

        self.module_config_links = {}

        # self.web_factory = Yombo_Site(self.webapp.resource(), None, logPath="/dev/null")
        self.web_factory = Yombo_Site(self.webapp.resource(),
                                      None,
                                      logPath=None)
        self.web_factory.setup_log_queue(self)
        self.web_factory.noisy = False  # turn off Starting/stopping message
        self.displayTracebacks = False

        self._display_how_to_access_at = 0  # When the display notice for how to access the web was shown.

        self.misc_wi_data["gateway_label"] = self._Configs.get2(
            "core", "label", "Yombo Gateway", False)
        self.misc_wi_data["operating_mode"] = self.operating_mode
        self.misc_wi_data["notifications"] = self._Notifications
        self.misc_wi_data[
            "notification_priority_map_css"] = NOTIFICATION_PRIORITY_MAP_CSS
        self.misc_wi_data["breadcrumb"] = []

        self.webapp.templates.globals["yombo"] = self
        self.webapp.templates.globals["_local_gateway"] = self._Gateways.local
        self.webapp.templates.globals["_amqp"] = self._AMQP
        self.webapp.templates.globals["_amqpyombo"] = self._AMQPYombo
        self.webapp.templates.globals["_authkeys"] = self._AuthKeys
        self.webapp.templates.globals["_atoms"] = self._Atoms
        self.webapp.templates.globals["_automation"] = self._Automation
        self.webapp.templates.globals["_cache"] = self._Cache
        self.webapp.templates.globals["_calllater"] = self._CallLater
        self.webapp.templates.globals["_commands"] = self._Commands
        self.webapp.templates.globals["_configs"] = self._Configs
        self.webapp.templates.globals["_crontab"] = self._CronTab
        self.webapp.templates.globals["_events"] = self._Events
        self.webapp.templates.globals["_devices"] = self._Devices
        self.webapp.templates.globals["_devicetypes"] = self._DeviceTypes
        self.webapp.templates.globals[
            "_downloadmodules"] = self._DownloadModules
        self.webapp.templates.globals["_gatewaycoms"] = self._GatewayComs
        self.webapp.templates.globals["_gateways"] = self._Gateways
        self.webapp.templates.globals["_gpg"] = self._GPG
        self.webapp.templates.globals["_inputtypes"] = self._InputTypes
        self.webapp.templates.globals["_intents"] = self._Intents
        self.webapp.templates.globals["_localize"] = self._Localize
        self.webapp.templates.globals["_locations"] = self._Locations
        self.webapp.templates.globals["_locations"] = self._Locations
        self.webapp.templates.globals["_modules"] = self._Modules
        self.webapp.templates.globals["_mqtt"] = self._MQTT
        self.webapp.templates.globals["_nodes"] = self._Nodes
        self.webapp.templates.globals["_notifiticaions"] = self._Notifications
        self.webapp.templates.globals["_users"] = self._Users
        self.webapp.templates.globals["_queue"] = self._Queue
        self.webapp.templates.globals["_scenes"] = self._Scenes
        self.webapp.templates.globals["_requests"] = self._Requests
        self.webapp.templates.globals["_sqldict"] = self._SQLDict
        self.webapp.templates.globals["_sslcerts"] = self._SSLCerts
        self.webapp.templates.globals["_states"] = self._States
        self.webapp.templates.globals["_statistics"] = self._Statistics
        self.webapp.templates.globals["_storage"] = self._Storage
        self.webapp.templates.globals["_tasks"] = self._Tasks
        self.webapp.templates.globals["_times"] = self._Times
        self.webapp.templates.globals["_variabledata"] = self._VariableData
        self.webapp.templates.globals["_variablefields"] = self._VariableFields
        self.webapp.templates.globals["_variablegroups"] = self._VariableGroups
        self.webapp.templates.globals["_validate"] = self._Validate
        self.webapp.templates.globals["_webinterface"] = self
        self.webapp.templates.globals["py_randint"] = randint
        self.webapp.templates.globals["py_time_time"] = time
        self.webapp.templates.globals["py_urllib_urlparse"] = urlparse
        self.webapp.templates.globals["py_urllib_urlunparse"] = urlunparse
        self.webapp.templates.globals["yombo_utils"] = yombo.utils
        self.webapp.templates.globals["misc_wi_data"] = self.misc_wi_data
        self.webapp.templates.globals["webinterface"] = self
        self.webapp.templates.globals["_location_id"] = None
        self.webapp.templates.globals["_area_id"] = None
        self.webapp.templates.globals["_location"] = None
        self.webapp.templates.globals["_area"] = None
        self.webapp.templates.globals["bg_image_id"] = lambda: int(time() / 300
                                                                   ) % 6
        self.webapp.templates.globals["get_alerts"] = self.get_alerts

        self._refresh_jinja2_globals_()
        self.starting = False
        self.start_web_servers()
        self.clean_idempotence_ids_loop = LoopingCall(
            self.clean_idempotence_ids)
        self.clean_idempotence_ids_loop.start(1806, False)

    def _refresh_jinja2_globals_(self, **kwargs):
        """
        Update various globals for the Jinja2 template.

        :return:
        """
        if self.operating_mode != "run":
            return
        self.webapp.templates.globals[
            "_location_id"] = self._Locations.location_id
        self.webapp.templates.globals["_area_id"] = self._Locations.area_id
        self.webapp.templates.globals["_location"] = self._Locations.location
        self.webapp.templates.globals["_area"] = self._Locations.area

    @inlineCallbacks
    def _start_(self, **kwargs):
        self._Notifications.add({
            "title": "System still starting",
            "message": "Still starting up. Please wait.",
            "source": "Web Interface Library",
            "persist": True,
            "priority": "high",
            "always_show": True,
            "always_show_allow_clear": False,
            "id": "webinterface:starting",
        })

        if self.operating_mode != "run":
            logger.warn(
                "First time running the gateway, it will take a while to build the frontend web server."
            )
            yield self.build_dist(
                verbose=True)  # Make all the JS and CSS files

        self._get_nav_side_items()
        self.webapp.templates.globals["_"] = _  # i18n

    def _started_(self, **kwargs):
        # if self.operating_mode != "run":
        self._display_how_to_access_at = int(time())
        self.display_how_to_access()
        self._Notifications.delete("webinterface:starting")
        self.web_interface_fully_started = True

        self.send_hook_listeners_ping_loop = LoopingCall(
            self.send_hook_listeners_ping_loop)
        self.send_hook_listeners_ping_loop.start(55, True)

    def clean_idempotence_ids(self):
        """
        Removes older idempotence keys.

        :return:
        """
        delete_time = int(time()) - 1800
        for key in self.idempotence.keys():
            if self.idempotence[key] < delete_time:
                del self.idempotence[key]

    def send_hook_listeners_ping_loop(self):
        route_api_v1_stream_broadcast(self, "ping", int(time()))

    def register_hook(self, name, thecallback):
        if name not in self.hook_listeners:
            self.hook_listeners[name] = []
        self.hook_listeners[name].append(thecallback)

    @inlineCallbacks
    def _yombo_universal_hook_(self, hook_name=None, **kwargs):
        """
        Implements the universal hook.

        :param kwargs:
        :return:
        """
        if hook_name in self.hook_listeners:
            for a_callback in self.hook_listeners[hook_name]:
                d = Deferred()
                d.addCallback(lambda ignored: maybeDeferred(
                    a_callback, self, hook_name=hook_name, **kwargs))
                d.addErrback(self.yombo_universal_hook_failure, hook_name,
                             a_callback)
                d.callback(1)
                yield d

    def yombo_universal_hook_failure(self, failure, hook_name, acallback):
        logger.warn(
            "---==(failure WI:universal hook for hook ({hook_name})==----",
            hook_name=hook_name)
        logger.warn("--------------------------------------------------------")
        logger.warn("{acallback}", acallback=acallback)
        logger.warn("{failure}", failure=failure)
        logger.warn("--------------------------------------------------------")
        raise RuntimeError(f"failure during module invoke for hook: {failure}")

    def check_have_required_nodes(self):
        try:
            node = yield self._Nodes.get("main_page", "webinterface_page")
        except KeyError as e:
            pass
            # add base node...

    def _configuration_set_(self, **kwargs):
        """
        Need to monitor if the web interface port has changed. This will restart the webinterface
        server if needed.

        :param kwargs: section, option(key), value
        :return:
        """
        section = kwargs["section"]
        option = kwargs["option"]
        value = kwargs["value"]

        if self.starting is True:
            return

        if section == "webinterface":
            if option == "nonsecure_port":
                self.change_ports(port_nonsecure=value)
            elif option == "secure_port":
                self.change_ports(port_secure=value)

    @inlineCallbacks
    def _unload_(self, **kwargs):
        if hasattr(self, "web_factory"):
            if self.web_factory is not None:
                yield self.web_factory.save_log_queue()

    @property
    def internal_url(self):
        """
        Returns the starting portion of the URL to this host.
        https://i.exmaple.yombo.net

        :return:
        """
        fqdn = self.fqdn()
        if fqdn is None:
            internal_hostname = self._Configs.get("core", "localipaddress_v4")
            return f"http://{internal_hostname}:{self.wi_port_nonsecure()}"
        else:
            return f"https://i.{fqdn}:{self.wi_port_secure()}"

    @property
    def external_url(self):
        """
        Returns the starting portion of the URL to this host.
        https://e.exmaple.yombo.net

        :return:
        """
        fqdn = self.fqdn()
        if fqdn is None:
            external_hostname = self._Configs.get("core",
                                                  "externalipaddress_v4")
            return f"https://{external_hostname}:{self.wi_port_secure()}"
        else:
            return f"https://e.{fqdn}:{self.wi_port_secure()}"

    def i18n(self, request):
        """
        Gets a translator based on the language the browser provides us.

        :param request: The browser request.
        :return:
        """
        locales = self._Localize.parse_accept_language(
            request.getHeader("accept-language"))
        locales_hash = yombo.utils.sha256_compact("".join(
            str(e) for e in locales))
        if locales_hash in self.translators:
            return self.translators[locales_hash]
        else:
            self.translators[locales_hash] = web_translator(self, locales)
        return self.translators[locales_hash]

    @inlineCallbacks
    def _get_nav_side_items(self, **kwargs):
        """
        Called before modules have their _prestart_ function called (after _load_).

        This implements the hook "webinterface_add_routes" and calls all libraries and modules. It allows libs and
        modules to add menus to the web interface and provide additional funcationality.

        **Usage**:

        .. code-block:: python

           def ModuleName_webinterface_add_routes(self, **kwargs):
               return {
                   "nav_side": [
                       {
                       "label1": "Tools",
                       "label2": "MQTT",
                       "priority1": 3000,
                       "priority2": 10000,
                       "icon": "fa fa-wrench fa-fw",
                       "url": "/tools/mqtt",
                       "tooltip": "",
                       "opmode": "run",
                       "cluster": "any",
                       },
                   ],
                   "routes": [
                       self.web_interface_routes,
                   ],
                   "configs" {
                        "settings_link": "/modules/tester/index",
                   },
               }

        """
        # first, lets get the top levels already defined so children don"t re-arrange ours.
        top_levels = {}
        add_on_menus = yield global_invoke_all(
            "_webinterface_add_routes_",
            called_by=self,
        )
        logger.debug("_webinterface_add_routes_ results: {add_on_menus}",
                     add_on_menus=add_on_menus)
        nav_side_menu = deepcopy(NAV_SIDE_MENU)
        for component, options in add_on_menus.items():
            if "nav_side" in options:
                for new_menu in options["nav_side"]:
                    if new_menu["label1"] in nav_side_menu:
                        the_index = nav_side_menu[new_menu["label1"]].index(
                            "bar")
                        new_menu["priority1"] = nav_side_menu[the_index][
                            "priority1"]
                    if "priority1" not in new_menu or isinstance(
                            new_menu["priority1"], int) is False:
                        new_menu["priority1"] = 1000
                    if "priority2" not in new_menu or isinstance(
                            new_menu["priority2"], int) is False:
                        new_menu["priority2"] = 100
                nav_side_menu = nav_side_menu + options["nav_side"]

        temp_list = sorted(NAV_SIDE_MENU,
                           key=itemgetter("priority1", "label1", "priority2"))
        for item in temp_list:
            label1 = item["label1"]
            if label1 not in temp_list:
                top_levels[label1] = item["priority1"]

        for component, options in add_on_menus.items():
            logger.debug("component: {component}, options: {options}",
                         component=component,
                         options=options)
            if "menu_priorities" in options:  # allow modules to change the ordering of top level menus
                for label, priority in options["menu_priorities"].items():
                    top_levels[label] = priority
            if "routes" in options:
                for new_route in options["routes"]:
                    new_route(self.webapp)
            if "configs" in options:
                if "settings_link" in options["configs"]:
                    self.module_config_links[component._module_id] = options[
                        "configs"]["settings_link"]

        # build menu tree
        self.misc_wi_data["nav_side"] = {}

        is_master = self.is_master
        # temp_list = sorted(nav_side_menu, key=itemgetter("priority1", "priority2", "label1"))
        temp_list = sorted(nav_side_menu,
                           key=itemgetter("priority1", "label1", "priority2",
                                          "label2"))
        for item in temp_list:
            if "cluster" not in item:
                item["cluster"] = "any"
            if item["cluster"] == "master" and is_master is not True:
                continue
            if item["cluster"] == "member" and is_master is True:
                continue
            item["label1_text"] = deepcopy(item["label1"])
            item["label2_text"] = deepcopy(item["label2"])
            label1 = "ui::navigation::" + yombo.utils.snake_case(
                item["label1"])
            item["label1"] = "ui::navigation::" + yombo.utils.snake_case(
                item["label1"])
            item["label2"] = "ui::navigation::" + yombo.utils.snake_case(
                item["label2"])
            if label1 not in self.misc_wi_data["nav_side"]:
                self.misc_wi_data["nav_side"][label1] = []
            self.misc_wi_data["nav_side"][label1].append(item)

        self.starting = False

    def add_alert(self,
                  session,
                  message,
                  level="info",
                  display_once=True,
                  deletable=True,
                  id=None):
        """
        Add an alert to the stack.
        :param level: info, warning, error
        :param message:
        :return:
        """
        id = session.add_alert(message, level, display_once, deletable, id)
        return id

    def get_alerts(self, session, autodelete=None):
        """
        Retrieve a list of alerts for display.
        """
        if session is None:
            return {}
        return session.get_alerts(autodelete)

    def get_template(self, request, template_path):
        request.webinterface.webapp.templates.globals[
            "_"] = request.webinterface.i18n(
                request)  # set in auth.update_request.
        return self.webapp.templates.get_template(template_path)

    def redirect(self, request, redirect_path):
        request.redirect(redirect_path)

    def _get_parms(self, request):
        return parse_qs(urlparse(request.uri).query)

    def request_get_default(self, request, name, default, offset=None):
        if offset == None:
            offset = 0
        try:
            return request.args.get(name)[offset]
        except:
            return default

    def home_breadcrumb(self, request):
        self.add_breadcrumb(request, "/", "Home")

    def add_breadcrumb(self,
                       request,
                       url=None,
                       text=None,
                       show=None,
                       style=None,
                       data=None):
        if hasattr(request, "breadcrumb") is False:
            request.breadcrumb = []
            self.misc_wi_data["breadcrumb"] = request.breadcrumb

        if show is None:
            show = True

        if style is None:
            style = "link"
        elif style == "select_groups":
            items = {}
            for option_label, option_data in data.items():
                items[option_label] = []
                for select_text, select_url in option_data.items():
                    selected = ""
                    option_style = "None"
                    if select_url.startswith("$"):
                        selected = "selected"
                        select_url = select_url[1:]
                    elif select_url.startswith("#"):
                        option_style = "divider"

                    items[option_label].append({
                        "option_style": option_style,
                        "text": select_text,
                        "url": select_url,
                        "selected": selected,
                    })
            data = items
        elif style == "select":
            items = []
            for select_text, select_url in data.items():
                selected = ""
                option_style = "None"
                if select_url.startswith("$"):
                    selected = "selected"
                    select_url = select_url[1:]
                elif select_url.startswith("#"):
                    option_style = "divider"

                items.append({
                    "option_style": option_style,
                    "text": select_text,
                    "url": select_url,
                    "selected": selected,
                })
            data = items

        hash = sha256(
            str(
                str(url) + str(text) + str(show) + str(style) +
                json.dumps(data)).encode()).hexdigest()
        breadcrumb = {
            "hash": hash,
            "url": url,
            "text": text,
            "show": show,
            "style": style,
            "data": data,
        }
        request.breadcrumb.append(breadcrumb)

    def setup_basic_filters(self):
        self.webapp.templates.filters["yes_no"] = yombo.utils.is_yes_no
        self.webapp.templates.filters["true_false"] = yombo.utils.is_true_false
        self.webapp.templates.filters["excerpt"] = yombo.utils.excerpt
        self.webapp.templates.filters["make_link"] = yombo.utils.make_link
        self.webapp.templates.filters[
            "status_to_string"] = converters.status_to_string
        self.webapp.templates.filters[
            "public_to_string"] = converters.public_to_string
        self.webapp.templates.filters[
            "epoch_to_string"] = converters.epoch_to_string
        self.webapp.templates.filters[
            "epoch_get_age"] = dt_util.get_age  # yesterday, 5 minutes ago, etc.
        self.webapp.templates.filters[
            "epoch_get_age_exact"] = dt_util.get_age_exact  # yesterday, 5 minutes ago, etc.
        self.webapp.templates.filters[
            "format_markdown"] = yombo.utils.format_markdown
        self.webapp.templates.filters[
            "hide_none"] = yombo.utils.display_hide_none
        self.webapp.templates.filters[
            "display_encrypted"] = self._GPG.display_encrypted
        self.webapp.templates.filters[
            "display_temperature"] = self._Localize.display_temperature
        self.webapp.templates.filters["json_human"] = yombo.utils.json_human
        self.webapp.templates.filters["yombo"] = self

    def restart(self, request, message=None, redirect=None):
        if message is None:
            message = ""
        if redirect is None:
            redirect = "/"

        page = self.get_template(request, self.wi_dir + "/pages/restart.html")
        reactor.callLater(0.3, self.do_restart)
        return page.render(message=message,
                           redirect=redirect,
                           uptime=str(self._Atoms["running_since"]))

    def do_restart(self):
        try:
            raise YomboRestart("Web Interface setup wizard complete.")
        except:
            pass

    def shutdown(self, request):
        page = self.get_template(request, self.wi_dir + "/pages/shutdown.html")
        # reactor.callLater(0.3, self.do_shutdown)
        return page.render()

    def do_shutdown(self):
        raise YomboCritical("Web Interface setup wizard complete.")
Beispiel #58
0
import time

from klein import Klein
from redis import Redis
import requests
from yarg.package import json2package


PYPI_URL = "https://pypi.python.org/pypi/%s/json"
SHIELD_URL = "http://img.shields.io/badge/%s-%s-%s.%s"
# SHIELD_URL = "http://localhost:9000/badge/%s-%s-%s.%s"  # pypip.in uses a local version of img.shields.io
FILE_CACHE = "/tmp/shields.py/"
CACHE_TIME = (60 * 60) * 24  # 24 hours
REDIS_EXPIRE = 60 * 10  # 10 minutes

app = Klein()
redis = Redis()


def format_number(singular, number):
    value = singular % {'value': number}
    # Get rid of the .0 but keep the other decimals
    return value.replace('.0', '')


intword_converters = (
    (3, lambda number: format_number('%(value).1fk', number)),
    (6, lambda number: format_number('%(value).1fM', number)),
    (9, lambda number: format_number('%(value).1fB', number)),
)
Beispiel #59
0
args = vars(parser.parse_args())

PORT = args["port"]
ADDRESS = args["ip"]
LOG = args["log"]

print(f"Connecting on {ADDRESS} on {PORT}")
"""
    Starting klein http-server
"""

from klein import Klein
from twisted.internet.defer import succeed
import wave

app = Klein()


@app.route("/transcribe", methods=["POST"])
def transcribe(request):

    fname = f"{str(time.time()).replace('.','-')}.wav"
    cvtfname = "cvt" + fname

    with open(fname, "wb") as fileOutput:
        fileOutput.write(request.content.read())
    """
        Currently, only 16-bit, 16 kHz, mono-channel WAVE audio files are supported in the Python client
    """

    converted = False
Beispiel #60
0
 def setUp(self):
     """
     Create an app and a resource wrapping that app for this test.
     """
     self.app = Klein()
     self.kr = self.app.resource()