forked from turtlesoupy/NPSGD
-
Notifications
You must be signed in to change notification settings - Fork 0
/
npsgd_web.py
executable file
·252 lines (202 loc) · 9.35 KB
/
npsgd_web.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
#!/usr/bin/python
# Author: Thomas Dimson [tdimson@gmail.com]
# Date: January 2011
# For distribution details, see LICENSE
"""Server for processing client (web-site) requests to NPSGD.
This module is meant to be run as a daemon process in conjunction with npsgd_queue
and npsgd_web. It acts as an HTTP server that serves up html for models
(e.g. http://localhost:8000/model/example) and can be proxied behind a server
or run as a web service.
"""
import os
import sys
import time
import logging
import functools
import threading
import tornado.web
import tornado.ioloop
import tornado.escape
import tornado.httpclient
import tornado.httpserver
import urllib
from optparse import OptionParser
from datetime import datetime
from npsgd import model_manager
from npsgd import model_parameters
from npsgd import ui_modules
from npsgd.model_manager import modelManager
from npsgd.model_task import ModelTask
from npsgd.model_parameters import ValidationError
from npsgd.config import config
#This is a simple cache so we don't overload the queue with unnecessary requests
#just to see it has workers
lastWorkerCheckSuccess = datetime(1,1,1)
class ClientModelRequest(tornado.web.RequestHandler):
"""HTTP handler for all model related requests."""
@tornado.web.asynchronous
def get(self, modelName):
"""Get handler to serve up the html for a specific model.
This method will either serve up an error if the queue is down or
the queue has no workers or return a nice view of the given model
using the model template. Requests to the queue are done
asynchronously to avoid blocking.
"""
global lastWorkerCheckSuccess
model = modelManager.getLatestModel(modelName)
td = datetime.now() - lastWorkerCheckSuccess
checkWorkers = (td.seconds + td.days * 24 * 3600) > config.keepAliveTimeout
if checkWorkers:
#check with queue to see if we have workers
http = tornado.httpclient.AsyncHTTPClient()
request = tornado.httpclient.HTTPRequest(
"http://%s:%s/client_queue_has_workers?secret=%s" % (config.queueServerAddress, config.queueServerPort, config.requestSecret),
method="GET")
callback = functools.partial(self.queueCallback, model=model)
http.fetch(request, callback)
else:
self.renderModel(model)
def renderModel(self, model):
"""Render the html associated with a given model object."""
self.render(config.modelTemplatePath, model=model, errorText=None)
def queueErrorRender(self, errorText, model):
self.render(config.modelErrorTemplatePath, errorText=errorText, model=model)
def queueCallback(self, response, model=None):
"""Asynchronous callback from the queue (for checking if it is up, or has workers).
This method will actually perform the rendering of the model if things
look good.
"""
global lastWorkerCheckSuccess
if response.error:
self.queueErrorRender("We are sorry. Our queuing server appears to be down at the moment, please try again later", model)
return
try:
json = tornado.escape.json_decode(response.body)
hasWorkers = json["response"]["has_workers"]
if hasWorkers:
lastWorkerCheckSuccess = datetime.now()
self.renderModel(model)
else:
self.queueErrorRender("We are sorry, our model worker machines appear to be down at the moment. Please try again later")
return
except KeyError:
logging.info("Bad response from queue server")
self.queueErrorRender("We are sorry. Our queuing server appears to be having issues communicating at the moment, please try again later")
return
@tornado.web.asynchronous
def post(self, modelName):
"""Post handler to actually create a model task with given parameters."""
modelVersion = self.get_argument("modelVersion")
try:
model = modelManager.getModel(modelName, modelVersion)
except KeyError:
self.queueErrorRender(
" We are sorry. Your model request did not match any models available."
" This is probably caused by an upgrade to the model versions, " +
" and will be fixed if you resubmit your request.")
return
try:
task = self.setupModelTask(model)
except ValidationError, e:
logging.exception(e)
self.render(config.modelTemplatePath, model=model, errorText=str(e))
return
except tornado.web.HTTPError, e:
logging.exception(e)
self.render(config.modelTemplatePath, model=model, errorText=None)
return
logging.info("Making async request to get confirmation number for task")
http = tornado.httpclient.AsyncHTTPClient()
request = tornado.httpclient.HTTPRequest(
"http://%s:%s/client_model_create" % (config.queueServerAddress, config.queueServerPort),
method="POST",
body=urllib.urlencode({
"secret": config.requestSecret,
"task_json": tornado.escape.json_encode(task.asDict())}))
http.fetch(request, self.confirmationNumberCallback)
def confirmationNumberCallback(self, response):
"""Asynchronous callback when model run is populated to confirmation queue."""
if response.error: raise tornado.web.HTTPError(500)
try:
json = tornado.escape.json_decode(response.body)
logging.info(json)
res = json["response"]
model = modelManager.getModel(res["task"]["modelName"], res["task"]["modelVersion"])
task = model.fromDict(res["task"])
code = res["code"]
except KeyError:
logging.info("Bad response from queue server")
raise tornado.web.HTTPError(500)
self.render(config.confirmTemplatePath, email=task.emailAddress, code=code)
def setupModelTask(self, model):
email = self.get_argument("email")
paramDict = {}
for param in model.parameters:
try:
argVal = self.get_argument(param.name)
except tornado.web.HTTPError:
argVal = param.nonExistValue()
value = param.withValue(argVal)
paramDict[param.name] = value.asDict()
task = model(email, 0, paramDict)
return task
class ClientBaseRequest(tornado.web.RequestHandler):
"""Index response handler (e.g. http://localhost:8000): render a tiny page for now."""
def get(self):
self.render(config.listModelsTemplatePath, modelNames=modelManager.modelNames())
class ClientConfirmRequest(tornado.web.RequestHandler):
"""HTTP confirmation code handler.
Requests to this server proxy to the queue for actual confirmation code handling.
The queue request is called asynchronously.
"""
@tornado.web.asynchronous
def get(self, confirmationCode):
http = tornado.httpclient.AsyncHTTPClient()
request = tornado.httpclient.HTTPRequest(
"http://%s:%s/client_confirm/%s?secret=%s" % (config.queueServerAddress, config.queueServerPort, \
confirmationCode, config.requestSecret))
logging.info("Asyncing a confirmation: '%s'", confirmationCode)
http.fetch(request, self.confirmationCallback)
def confirmationCallback(self, response):
if response.error: raise tornado.web.HTTPError(404)
json = tornado.escape.json_decode(response.body)
res = json["response"]
if res == "okay":
self.render(config.confirmedTemplatePath)
elif res == "already_confirmed":
self.render(config.alreadyConfirmedTemplatePath)
else:
logging.info("Bad response from queue server: %s", res)
raise tornado.web.HTTPError(500)
def setupClientApplication():
appList = [
(r"/", ClientBaseRequest),
(r"/confirm_submission/(\w+)", ClientConfirmRequest),
(r"/models/(.*)", ClientModelRequest)
]
settings = {
"static_path": os.path.join(os.path.dirname(__file__), "static"),
"ui_modules": ui_modules,
"template_path": config.htmlTemplateDirectory
}
return tornado.web.Application(appList, **settings)
def main():
parser = OptionParser()
parser.add_option('-c', '--config', dest='config',
help="Config file", default="config.cfg")
parser.add_option('-p', '--client-port', dest='port',
help="Http port (for serving html)", default=8000, type="int")
parser.add_option('-l', '--log-filename', dest='log',
help="Log filename (use '-' for stderr)", default="-")
(options, args) = parser.parse_args()
config.loadConfig(options.config)
config.setupLogging(options.log)
model_manager.setupModels()
model_manager.startScannerThread()
clientHTTP = tornado.httpserver.HTTPServer(setupClientApplication())
clientHTTP.listen(options.port)
logging.info("NPSGD Web Booted up, serving on port %d", options.port)
print >>sys.stderr, "NPSGD web server running on port %d" % options.port
tornado.ioloop.IOLoop.instance().start()
if __name__ == "__main__":
main()