def test_register_account_api_endpoints(self):
app = klein.Klein()
auth = DummyAuthProvider()
result = ['foobar']
with mock.patch('stethoscope.plugins.utils.instantiate_plugins') as \
mock_instantiate_plugins:
mock_instantiate_plugins.return_value = stevedore.ExtensionManager.make_test_instance(
[get_mock_ext(result)])
stethoscope.api.factory.register_account_api_endpoints(
app, config, auth)
# see klein's test_app.py
self.assertEqual(
app.url_map.bind('account-foo-email').match(
"/account/foo/email/[email protected]"),
('account-foo-email', {
'email': "*****@*****.**"
}))
self.assertEqual(
app.url_map.bind('accounts-merged').match(
"/accounts/merged/[email protected]"),
('accounts-merged', {
'email': "*****@*****.**"
}))
self.assertEqual(len(app.endpoints), 2)
self.check_result(app, b'/account/foo/email/[email protected]', result)
self.check_result(app, b'/accounts/merged/[email protected]', [result])
class Outline(object):
app = klein.Klein()
def __init__(self, store):
self.store = store
@app.route('/toc', methods=['GET'])
def toc(self, request):
ret = {}
for child in self.store.children():
content = child.getContent()
parsed = json.loads(content)
logicalName = parsed['logicalName']
realName = child.basename()
ret[logicalName] = realName
return json.dumps(ret)
@app.route('/children/<child>', methods=['PUT'])
def add(self, request, child):
fp = self.store.child(child)
if fp.exists():
raise ValueError("cannot modify")
request.content.seek(0, 0)
content = json.loads(request.content.read())
parent = content.get('parent')
if parent:
parentFP = self.store.child(parent)
parentContents = json.loads(parentFP.getContent())
lastID = parentContents['lastID'] = parentContents.get(
'lastID', 0) + 1
parentFP.setContent(json.dumps(parentContents))
child['logicalName'] = parent['logicalName'] + '.' + lastID
fp.setContent(json.dumps(child))
return 'Done'
def test_add_route(self):
app = klein.Klein()
auth = DummyAuthProvider()
result = ['foobar']
ext = get_mock_ext(result)
callback = mock.Mock(side_effect=lambda x: x)
stethoscope.api.factory._add_route(ext,
app,
auth,
'resource',
'email',
callbacks=[callback])
# see klein's test_app.py
self.assertEqual(
app.url_map.bind('resource-foo-email').match(
"/resource/foo/email/[email protected]"),
('resource-foo-email', {
'email': "*****@*****.**"
}))
self.assertEqual(len(app.endpoints), 1)
self.check_result(app, b'/resource/foo/email/[email protected]', result,
callback)
def setUp(self):
self.csrf = stethoscope.csrf.CSRFProtection({})
self.app = klein.Klein()
@self.app.route("/")
@self.csrf.csrf_protect
def endpoint(request):
return mock.sentinel.DEFAULT
def setUp(self):
self.app = app = klein.Klein()
self.auth = auth = DummyAuthProvider()
self.config = config = {
'DEBUG': True,
'TESTING': True,
}
stethoscope.api.factory.register_error_handlers(app, config, auth)
class SlowIncrementWebServer(object):
application = klein.Klein()
def __init__(self, reactor):
self._reactor = reactor
@application.route('/<int:amount>')
def slow_increment(self, request, amount):
new_amount = amount + 1
message = f'Hello! Your new amount is: {new_amount}'
return task.deferLater(self._reactor, 1.0, str.encode, message,
'ascii')
def test_register_notification_api_endpoints(self):
app = klein.Klein()
auth = DummyAuthProvider()
result_foo = [{'_source': {'event_timestamp': 0}}]
result_bar = [{'_source': {'event_timestamp': 1}}]
mock_hook = mock.Mock()
mock_hook.obj.transform.side_effect = lambda x: x
mock_hook_manager = stevedore.ExtensionManager.make_test_instance(
[mock_hook])
mock_extension_manager = stevedore.ExtensionManager.make_test_instance(
[get_mock_ext(result_foo, 'foo'),
get_mock_ext(result_bar, 'bar')])
with mock.patch('stethoscope.plugins.utils.instantiate_plugins') as \
mock_instantiate_plugins:
mock_instantiate_plugins.side_effect = [
mock_extension_manager, mock_hook_manager
]
stethoscope.api.factory.register_notification_api_endpoints(
app, config, auth)
# see klein's test_app.py
adapter_foo = app.url_map.bind('notifications-foo-email')
self.assertEqual(
adapter_foo.match("/notifications/foo/email/[email protected]"),
('notifications-foo-email', {
'email': "*****@*****.**"
}))
adapter_bar = app.url_map.bind('notifications-bar-email')
self.assertEqual(
adapter_bar.match("/notifications/bar/email/[email protected]"),
('notifications-bar-email', {
'email': "*****@*****.**"
}))
self.assertEqual(
app.url_map.bind('notifications-merged').match(
"/notifications/merged/[email protected]"),
('notifications-merged', {
'email': "*****@*****.**"
}))
self.assertEqual(len(app.endpoints), 3)
self.check_result(app, b'/notifications/foo/email/[email protected]',
result_foo)
self.check_result(app, b'/notifications/bar/email/[email protected]',
result_bar)
self.check_result(app, b'/notifications/merged/[email protected]',
result_bar + result_foo)
def test_route_token_required(self):
app = klein.Klein()
@app.route("/")
@self.auth.token_required
def endpoint(request, userinfo):
return userinfo
mock_request = mock.create_autospec(twisted.web.http.Request)
mock_request.getCookie.return_value = None
returned = app.execute_endpoint("endpoint", mock_request)
self.assertEqual(returned, self.userinfo)
class DocStore(object):
documents = attr.ib(default=attr.Factory(dict))
users = attr.ib(default=attr.Factory(dict))
def whoami(self, request):
return IUser(request.getSession())
app = klein.Klein()
@app.route('/api/doc', methods=['GET'])
def api_docs(self, request):
return asJSON(request, self.documents.keys())
@app.route('/api/whoami', methods=['GET'])
def api_whoami_get(self, request):
return asJSON(request, {'user': self.whoami(request).name})
@app.route('/api/whoami', methods=['PUT'])
def api_whoami_set(self, request):
body = json.load(request.content)
if body['user'] in self.users:
request.setResponseCode(409)
return asJSON(request, {})
user = self.whoami(request)
user.name = body['user']
self.users[user.name] = user
return asJSON(request, {'user': user.name})
@app.route('/api/doc', methods=['POST'])
def api_doc_create(self, request):
body = json.load(request.content)
doc = Document(content=body['content'],
id=base64.b64encode(os.urandom(9)))
self.documents[doc.id] = doc
return asJSON(request, {'created': True, 'id': doc.id})
@app.route('/api/doc/<docId>')
def api_docs(self, request, docId):
doc = self.documents.get(docId)
if doc is None:
request.setResponseCode(404)
return asJSON(request, {})
else:
user = self.whoami(request)
if doc.owner is None:
doc.owner = user
doc.users.add(user)
return asJSON(request, doc.asJSON(user))
def create_app():
config = get_config()
config['LOGBOOK'].push_application()
app = klein.Klein()
auth = stethoscope.auth.KleinAuthProvider(config)
csrf = stethoscope.csrf.CSRFProtection(config)
register_error_handlers(app, config, auth)
register_endpoints(app, config, auth, csrf)
logger.info("Shields up, weapons online.")
return app
class TargetCounter(object):
engine: Any
app = klein.Klein()
@app.route('/')
def count(self, request):
return str(len(list(self.engine.current_scene.get(tag='target'))))
@classmethod
def web_server(cls, reactor, engine, description):
ep = endpoints.serverFromString(reactor, description)
counter = cls(engine)
return ep.listen(Site(counter.app.resource()))
class Operations(object):
app = klein.Klein()
clock = attr.ib()
@app.route('/multiply')
def multiply(self, request):
a, b = _get_numbers(request)
res = task.deferLater(self.clock, 5, operator.mul, a, b)
return _send_number(request, res)
@app.route('/negate')
def negate(self, request):
a, = _get_numbers(request)
res = task.deferLater(self.clock, 3, operator.neg, a)
return _send_number(request, res)
def create_app():
config = get_config()
config['LOGBOOK'].push_application()
if "PLUGINS" not in config or len(config["PLUGINS"]) < 1:
logger.warn("Missing or invalid PLUGINS configuration!")
app = klein.Klein()
auth = stethoscope.auth.KleinAuthProvider(config)
csrf = stethoscope.csrf.CSRFProtection(config)
register_error_handlers(app, config, auth)
register_endpoints(app, config, auth, csrf)
logger.info("Shields up, weapons online.")
return app
def test_route_match_required(self):
app = klein.Klein()
kr = KleinResource(app)
@app.route("/api/<string:email>", endpoint="endpoint")
@self.auth.match_required
def endpoint(request, userinfo, email):
logger.debug(
"in endpoint: request={!r}, userinfo={!r}, email={!r}",
request, userinfo, email)
return json.dumps(userinfo)
request = requestMock(b"/api/[email protected]")
deferred = _render(kr, request)
self.assertEqual(self.successResultOf(deferred), None)
self.assertEqual(request.getWrittenData(),
json.dumps(self.userinfo).encode('ascii'))
def setUp(self):
super(AppTestCase, self).setUp()
self.app = klein.Klein()
self.kr = KleinResource(self.app)
@self.app.route("/")
@self.auth.token_required
def token_required_endpoint(request, userinfo):
return userinfo
self.token_required_endpoint = token_required_endpoint
@self.app.route("/api/<string:email>")
@self.auth.match_required
def match_required_endpoint(request, userinfo, email):
logger.debug(
"in endpoint: request={!r}, userinfo={!r}, email={!r}",
request, userinfo, email)
return json.dumps(userinfo)
self.match_required_endpoint = match_required_endpoint
def test_register_device_api_endpoints(self):
app = klein.Klein()
auth = DummyAuthProvider()
result = ['foobar']
with mock.patch('stethoscope.plugins.utils.instantiate_plugins') as \
mock_instantiate_plugins:
mock_instantiate_plugins.return_value = stevedore.ExtensionManager.make_test_instance(
[get_mock_ext(result)])
stethoscope.api.factory.register_device_api_endpoints(app, config, auth)
# see klein's test_app.py
self.assertEqual(
app.url_map.bind('devices-foo-email').match("/devices/foo/email/[email protected]"),
('devices-foo-email', {'email': "*****@*****.**"})
)
self.assertEqual(
app.url_map.bind('devices-foo-macaddr').match("/devices/foo/macaddr/de:ca:fb:ad:00:00"),
('devices-foo-macaddr', {'macaddr': "de:ca:fb:ad:00:00"}))
self.assertEqual(app.url_map.bind('devices-foo-serial').match("/devices/foo/serial/0xDEADBEEF"),
('devices-foo-serial', {'serial': "0xDEADBEEF"}))
self.assertEqual(app.url_map.bind('devices-email').match("/devices/email/[email protected]"),
('devices-email', {'email': "*****@*****.**"}))
self.assertEqual(app.url_map.bind('devices-serial').match("/devices/serial/0xDECAFBAD"),
('devices-serial', {'serial': "0xDECAFBAD"}))
self.assertEqual(app.url_map.bind('devices-macaddr').match(
"/devices/macaddr/DE:CA:FB:AD:00:00"),
('devices-macaddr', {'macaddr': "DE:CA:FB:AD:00:00"}))
self.assertEqual(app.url_map.bind('devices-staged').match("/devices/staged/[email protected]"),
('devices-staged', {'email': "*****@*****.**"}))
self.assertEqual(app.url_map.bind('devices-merged').match("/devices/merged/[email protected]"),
('devices-merged', {'email': "*****@*****.**"}))
self.assertEqual(len(app.endpoints), 8)
self.check_result(app, b'/devices/foo/email/[email protected]', result)
from twisted.internet import task
from twisted.internet import reactor
import klein
application = klein.Klein()
@application.route('/<int:amount>')
def slow_increment(request, amount):
new_amount = amount + 1
message = f'Hello! Your new amount is: {new_amount}'
return task.deferLater(reactor, 1.0, str.encode, message, 'ascii')
if __name__ == '__main__':
application.run('localhost', 8080)
class InfobobWebUI(object):
app = klein.Klein()
def __init__(self, loader, dbpool):
self.loader = loader
self.dbpool = dbpool
@app.route('/bans')
@inlineCallbacks
def bans(self, request):
bans = yield self.dbpool.get_active_bans()
bans = itertools.groupby(bans, operator.itemgetter(0))
renderTemplate(request,
self.loader.load('bans.html'),
bans=bans,
show_unset=False,
show_recent_expiration=False)
@app.route('/bans/expired')
@app.route('/bans/expired/<int:count>')
@inlineCallbacks
def expiredBans(self, request, count=10):
bans = yield self.dbpool.get_recently_expired_bans(count)
bans.sort(key=operator.itemgetter(0, 7))
bans = itertools.groupby(bans, operator.itemgetter(0))
renderTemplate(request,
self.loader.load('bans.html'),
bans=bans,
show_unset=True,
show_recent_expiration=True)
@app.route('/bans/all')
@inlineCallbacks
def allBans(self, request):
bans = yield self.dbpool.get_all_bans()
bans = itertools.groupby(bans, operator.itemgetter(0))
renderTemplate(request,
self.loader.load('bans.html'),
bans=bans,
show_unset=True,
show_recent_expiration=False)
@app.route('/bans/edit/<rowid>/<auth>', methods=['GET', 'HEAD'])
@inlineCallbacks
def editBan(self, request, rowid, auth):
ban = yield self.dbpool.get_ban_with_auth(rowid, auth)
renderTemplate(request,
self.loader.load('edit_ban.html'),
ban=ban,
message=None)
@app.route('/bans/edit/<rowid>/<auth>', methods=['POST'])
@inlineCallbacks
def postEditBan(self, request, rowid, auth):
ban = yield self.dbpool.get_ban_with_auth(rowid, auth)
_, _, _, _, _, expire_at, reason, _, _ = ban
if 'expire_at' in request.args:
raw_expire_at = request.args['expire_at'][0]
if raw_expire_at == 'never':
expire_at = None
else:
try:
expire_at = parse_time_string(raw_expire_at)
except ValueError:
# This will cause the ban reason in the form to be the old
# one (from the DB), not very user-friendly... but it
# prevents the exception from breaking the page.
message = (
'Invalid expiration timestamp or relative date {0!r}'
).format(raw_expire_at)
renderTemplate(request,
self.loader.load('edit_ban.html'),
ban=ban,
message=message)
return
if 'reason' in request.args:
reason = request.args['reason'][0]
yield self.dbpool.update_ban_by_rowid(rowid, expire_at, reason)
ban = ban[:5] + (expire_at, reason) + ban[7:]
renderTemplate(request,
self.loader.load('edit_ban.html'),
ban=ban,
message='ban details updated')
class DownloadEFolder(object):
app = klein.Klein()
def __init__(self, logger, download_database, storage_path, fernet,
vbms_client, queue, env_name):
self.logger = logger
self.download_database = download_database
self.storage_path = storage_path
self.fernet = fernet
self.vbms_client = vbms_client
self.queue = queue
self.env_name = env_name
self.jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader(
FilePath(__file__).parent().parent().child("templates").path, ),
autoescape=True)
self.document_types = DeferredValue()
@classmethod
def from_config(cls, reactor, logger, queue, config_path):
with open(config_path) as f:
config = yaml.safe_load(f)
# TODO: bump this once alchimia properly handles pinning
thread_pool = ThreadPool(minthreads=1, maxthreads=1)
thread_pool.start()
reactor.addSystemEventTrigger('during', 'shutdown', thread_pool.stop)
return cls(
logger,
DownloadDatabase(reactor, thread_pool, config["db"]["uri"]),
FilePath(config["storage"]["filesystem"]),
fernet.MultiFernet(
[fernet.Fernet(key) for key in config["encryption_keys"]]),
VBMSClient(
reactor,
connect_vbms_path=config["connect_vbms"]["path"],
bundle_path=config["connect_vbms"]["bundle_path"],
endpoint_url=config["vbms"]["endpoint_url"],
keyfile=config["vbms"]["keyfile"],
samlfile=config["vbms"]["samlfile"],
key=config["vbms"].get("key"),
keypass=config["vbms"]["keypass"],
ca_cert=config["vbms"].get("ca_cert"),
client_cert=config["vbms"].get("client_cert"),
),
queue,
config["env"],
)
@classmethod
def create_demo(cls, reactor, logger):
return cls(logger=logger,
download_database=DemoMemoryDownloadDatabase(),
storage_path=None,
fernet=None,
vbms_client=None,
queue=None,
env_name="demo")
def render_template(self, template_name, data={}):
t = self.jinja_env.get_template(template_name)
return t.render(dict(data, env=self.env_name))
@inlineCallbacks
def start_download(self, file_number, request_id):
logger = self.logger.bind(file_number=file_number,
request_id=request_id)
logger.emit("list_documents.start")
try:
documents = yield self.vbms_client.list_documents(
logger, file_number)
except VBMSError as e:
logger.bind(
stdout=e.stdout,
stderr=e.stderr,
exit_code=e.exit_code,
).emit("list_documents.error")
yield self.download_database.mark_download_errored(
logger, request_id)
else:
logger.emit("list_documents.success")
documents = [
Document.from_json(request_id, doc) for doc in documents
]
yield self.download_database.create_documents(logger, documents)
for doc in documents:
self.queue.put(
functools.partial(self.start_file_download, logger, doc))
yield self.download_database.mark_download_manifest_downloaded(
logger, request_id)
@inlineCallbacks
def start_file_download(self, logger, document):
logger = logger.bind(document_id=document.document_id)
logger.emit("get_document.start")
try:
contents = yield self.vbms_client.fetch_document_contents(
logger, str(document.document_id))
except VBMSError as e:
logger.bind(
stdout=e.stdout,
stderr=e.stderr,
exit_code=e.exit_code,
).emit("get_document.error")
yield self.download_database.mark_document_errored(
logger, document)
else:
logger.emit("get_document.success")
target = self.storage_path.child(str(uuid.uuid4()))
target.setContent(self.fernet.encrypt(contents))
yield self.download_database.set_document_content_location(
logger, document, target.path)
@inlineCallbacks
def start_fetch_document_types(self):
document_types = yield self.vbms_client.get_document_types(self.logger)
self.document_types.completed(
{int(c["type_id"]): c["description"]
for c in document_types})
@inlineCallbacks
def queue_pending_work(self):
downloads, documents = yield self.download_database.get_pending_work(
self.logger)
for download in downloads:
self.queue.put(
functools.partial(
self.start_download,
download.file_number,
download.request_id,
))
for document in documents:
# TODO: self.logger doesn't include any info about the
# DownloadStatus. When triggered from the "usual" path it has
# ``file_number`` and ``request_id`` keys.
self.queue.put(
functools.partial(self.start_file_download, self.logger,
document))
@app.route("/")
@instrumented_route
def root(self, request):
request.redirect("/efolder-express/")
return succeed(None)
@app.route("/efolder-express/")
@instrumented_route
def index(self, request):
return self.render_template("index.html")
@app.route("/efolder-express/download/", methods=["POST"])
@instrumented_route
@inlineCallbacks
def download(self, request):
file_number = request.args["file_number"][0]
file_number = file_number.replace("-", "").replace(" ", "")
request_id = str(uuid.uuid4())
yield self.download_database.create_download(self.logger, request_id,
file_number)
self.queue.put(
functools.partial(self.start_download, file_number, request_id))
request.redirect("/efolder-express/download/{}/".format(request_id))
returnValue(None)
@app.route("/efolder-express/download/<request_id>/")
@instrumented_route
@inlineCallbacks
def download_status(self, request, request_id):
download = yield self.download_database.get_download(
self.logger, request_id=request_id)
returnValue(self.render_template("download.html",
{"status": download}))
@app.route("/efolder-express/download/<request_id>/json/")
@instrumented_route
@inlineCallbacks
def download_status_json(self, request, request_id):
download = yield self.download_database.get_download(
self.logger, request_id=request_id)
html = self.render_template("_download_status.html", {
"status": download,
})
request.setHeader("Content-Type", "application/json")
returnValue(
json.dumps({
"completed": download.completed or download.state == "ERRORED",
"html": html,
}))
@app.route("/efolder-express/download/<request_id>/zip/")
@instrumented_route
@inlineCallbacks
def download_zip(self, request, request_id):
download = yield self.download_database.get_download(
self.logger, request_id=request_id)
assert download.completed
self.logger.bind(
request_id=request_id,
file_number=download.file_number,
).emit("download")
document_types = yield self.document_types.wait()
path = download.build_zip(self.jinja_env, self.fernet, document_types)
request.setHeader(
"Content-Disposition",
"attachment; filename={}-eFolder.zip".format(download.file_number))
resource = File(path, defaultType="application/zip")
resource.isLeaf = True
request.notifyFinish().addBoth(lambda *args, **kwargs: os.remove(path))
returnValue(resource)
