def start(self, num_processes: Optional[int] = 1, max_restarts: int = None) -> None:
"""Starts this server in the `.IOLoop`.
By default, we run the server in this process and do not fork any
additional child process.
If num_processes is ``None`` or <= 0, we detect the number of cores
available on this machine and fork that number of child
processes. If num_processes is given and > 1, we fork that
specific number of sub-processes.
Since we use processes and not threads, there is no shared memory
between any server code.
Note that multiple processes are not compatible with the autoreload
module (or the ``autoreload=True`` option to `tornado.web.Application`
which defaults to True when ``debug=True``).
When using multiple processes, no IOLoops can be created or
referenced until after the call to ``TCPServer.start(n)``.
Values of ``num_processes`` other than 1 are not supported on Windows.
The ``max_restarts`` argument is passed to `.fork_processes`.
.. versionchanged:: 6.0
Added ``max_restarts`` argument.
"""
assert not self._started
self._started = True
if num_processes != 1:
process.fork_processes(num_processes, max_restarts)
sockets = self._pending_sockets
self._pending_sockets = []
self.add_sockets(sockets)
def main():
script_path = os.path.dirname(os.path.realpath(__file__))
config_path = os.path.join(script_path, 'etc/harvest.cfg')
config = ConfigParser()
config.read(config_path)
sockets = bind_sockets(config.get('server', 'port'),
config.get('server', 'address'))
fork_processes(config.getint('server', 'instances'))
datastore = DataStore(config.get('datastore', 'host'),
config.getint('datastore', 'port'),
config.get('datastore', 'username'),
config.get('datastore', 'password'),
config.get('datastore', 'database'))
app = Application([(r"/rpc/store", Handler,
{'datastore': datastore,
'api_key': config.get('server', 'api_key')})])
server = HTTPServer(app,
no_keep_alive=config.get('server', 'no_keep_alive'),
ssl_options={
'certfile': config.get('server', 'certfile'),
'keyfile': config.get('server', 'keyfile')})
server.add_sockets(sockets)
IOLoop.instance().start()
def main():
options.parse_command_line()
_port = options.options.port
_process_num = options.options.process
_debug_level = options.options.debug * 10
process.fork_processes(_process_num, max_restarts=3)
process_port = _port + process.task_id()
process_debug = _process_num <= 1 and _debug_level < 30
print('Service Running on %d ...' % process_port)
app = web.Application((
(r'/', views.base.IndexHandler),
(r'/home', views.base.HomeHandler),
(r'/auth/redirect', views.auth.OAuth2RedirectHandler),
(r'/auth/revoke', views.auth.OAuth2RevokeHandler),
(r'/auth/authorize', views.auth.OAuth2AuthorizeHandler),
(r'/auth/info', views.auth.OAuth2InfoHandler),
(r'/user/info', views.rest.UserInfoHandler),
(r'/user/option', views.rest.UserOptionHandler),
(r'/weibo/public', views.rest.WeiboPublicHandler),
(r'/weibo/sync', views.rest.WeiboSyncHandler),
(r'/weibo/query', views.rest.WeiboQueryHandler),
(r'/weibo/redirect', views.rest.WeiboRedirectHandler),
(r'/emotion/query', views.rest.EmotionQueryHandler),
), debug=process_debug, cookie_secret=setting.COOKIE_SECRET)
app.listen(process_port, xheaders=True)
loop = ioloop.IOLoop.instance()
loop.start()
def main():
""" entry """
try:
conf = __import__('conf')
except ImportError as e:
app_log.critical("Unable to load site config. ({})".format(e))
raise SystemExit()
parse_command_line()
if options.debug:
app_log.setLevel(logging.DEBUG)
if not options.debug:
fork_processes(None)
options.port += task_id() or 0
if not os.path.isdir(conf.app_path):
app_log.critical("{p} isn't accessible, maybe "
"create it?".format(p=conf.app_path))
raise SystemExit()
app_log.debug("Starting {name} on port {port}".format(name=conf.name,
port=options.port))
# initialize the application
tornado.httpserver.HTTPServer(Application(options,
conf)).listen(options.port,
'0.0.0.0')
ioloop = tornado.ioloop.IOLoop.instance()
if options.debug:
tornado.autoreload.start(ioloop)
# enter the Tornado IO loop
ioloop.start()
def start(self, num_processes=1):
"""Starts this server in the IOLoop.
By default, we run the server in this process and do not fork any
additional child process.
If num_processes is ``None`` or <= 0, we detect the number of cores
available on this machine and fork that number of child
processes. If num_processes is given and > 1, we fork that
specific number of sub-processes.
Since we use processes and not threads, there is no shared memory
between any server code.
Note that multiple processes are not compatible with the autoreload
module (or the ``debug=True`` option to `tornado.web.Application`).
When using multiple processes, no IOLoops can be created or
referenced until after the call to ``TCPServer.start(n)``.
"""
assert not self._started
self._started = True
if num_processes != 1:
process.fork_processes(num_processes)
sockets = self._pending_sockets
self._pending_sockets = []
self.add_sockets(sockets)
def start(self, num_processes=1):
r"""Starts the server in IOLoop."""
assert not self._started
self._started = True
if num_processes != 1:
process.fork_processes(num_processes)
sockets, self._pending_sockets = self._pending_sockets, []
self.add_sockets(sockets)
def main():
try:
options,datafile_json=optparse_lib().parse_args()
cmdstr="""nohup python easystatserver.py > /dev/null 2>&1 &"""
#cmdstr="""nohup ./easystatserver > /dev/null 2>&1 &"""
#status,output=cmd_execute(cmdstr)
import os
os.system(cmdstr)
#print(u"start benchmark test...")
if(options.processnum != -1):
process.fork_processes(options.processnum)
'''logging.basicConfig(level=logging.DEBUG,format='[%(levelname)s] (%(asctime)s) <%(message)s>',datefmt='%a,%Y-%m-%d %H:%M:%S',
filename="./log/process."+str(tornado.process.task_id())+".log",filemode='w')'''
logging.basicConfig(level=logging.ERROR,
format='[%(levelname)s] [%(asctime)s] [%(filename)s-line:%(lineno)d] [%(funcName)s-%(threadName)s] %(message)s',
datefmt='%a,%Y-%m-%d %H:%M:%S',
filename="./log/easyhttpbenchmark.log",
filemode='a')
easyhttpbc=easyhttpbenchmark(options.maxclientnum,options.clientnum,options.testtime,options.flag,datafile_json)
easyhttpbc.benchmark_test()
#print(u"benchmark test end...")
except Exception as e:
logging.error(str(e))
try:
from xmlrpclib import ServerProxy
cfg_json=json.load(open("./conf/easyhttpbenchmark.conf", "r"),encoding='utf-8')
stat_rpc_server=cfg_json['stat_rpc_server']
stat_rpc_port=cfg_json['stat_rpc_port']
svr=ServerProxy("http://"+stat_rpc_server+":"+stat_rpc_port)
'''print("total_req_cnt:"+str(easyhttpbc.total_req_cnt))
print("total_res_cnt:"+str(easyhttpbc.total_res_cnt))
print("total_err_cnt:"+str(easyhttpbc.total_err_cnt))
print("total_nul_cnt:"+str(easyhttpbc.total_nul_cnt))'''
import multiprocessing
cpu_count=multiprocessing.cpu_count()
if(options.processnum != 0):
svr.stat_maxclientnum(options.processnum*options.maxclientnum)
svr.stat_clientnum(options.processnum*options.clientnum)
else:
svr.stat_maxclientnum(cpu_count*options.maxclientnum)
svr.stat_clientnum(cpu_count*options.clientnum)
svr.set_test_time(easyhttpbc.testtime)
svr.stat_total_req_cnt(easyhttpbc.total_req_cnt)
svr.stat_total_res_cnt(easyhttpbc.total_res_cnt)
svr.stat_total_err_cnt(easyhttpbc.total_err_cnt)
svr.stat_total_nul_cnt(easyhttpbc.total_nul_cnt)
svr.stat_total_below_10(easyhttpbc.below_10)
svr.stat_total_between_10_20(easyhttpbc.between_10_20)
svr.stat_total_between_20_30(easyhttpbc.between_20_30)
svr.stat_total_over_30(easyhttpbc.over_30)
svr.stat_total_res_time(easyhttpbc.total_res_time)
except Exception as e:
logging.error(str(e))
def server_forever(self, *args, **kwargs):
try:
if self._multiprocess:
info('starting tornado server in multi-process mode')
fork_processes(self._num_processes)
else:
info('starting tornado server in single-process mode')
self._server = HTTPServer(self._app)
self._server.add_sockets(self._sockets)
IOLoop.instance().start()
except Exception, e:
error("exception in serve_forever: %s", e)
def main():
parse_command_line()
settings = {
'debug': options.debug
}
sockets = bind_sockets(options.port)
fork_processes(0)
application = router(settings)
_http_server = HTTPServer(application)
_http_server.add_sockets(sockets)
IOLoop.current().start()
def main():
options,DataFile=OptParseLib().parse_args()
ConfFile=json.load(open(ConfFilePath, "r"),encoding='utf-8')
#print(ConfFile)
#print(options.ProcessNUM,options.ClientNUM,options.TEST_TIME,DataFile)
print(u"Start Benchmark Test...")
if options.ProcessNUM != -1:
process.fork_processes(options.ProcessNUM)
logging.basicConfig(level=logging.DEBUG,format='[%(levelname)s] (%(asctime)s) <%(message)s>',datefmt='%a,%Y-%m-%d %H:%M:%S',
filename=ConfFile["LogFilePath"]+str(tornado.process.task_id())+".log",filemode='w')
easyBC=easyBenchmarkTesttool(options.ClientNUM,options.TEST_TIME,DataFile)
easyBC.benchmark_test()
print(u"Benchmark Test End...")
def run(self):
if options.debug:
app_log.setLevel(logging.DEBUG)
if not options.debug:
fork_processes(None)
options.port += task_id() or 0
app_log.debug("Starting %s on port %s" % (cfg.platform_name, options.port))
# initialize the application
tornado.httpserver.HTTPServer(Application(self.commons)).listen(options.port, '0.0.0.0')
ioloop = tornado.ioloop.IOLoop.instance()
if options.debug:
tornado.autoreload.start(ioloop)
# enter the Tornado IO loop
ioloop.start()
def fork():
"""
Fork the processes off, set process titles (master, worker) and return
and ioloop.
Returns an instance of tornado.ioloop.IOLoop
"""
# Set the custom process title of the master
set_process_title()
# Fork and create the ioloop
options.workers = opts.workers()
process.fork_processes(options.workers)
io_loop = ioloop.IOLoop.instance()
# Set the custom process title of the workers
set_process_title()
return io_loop
def main():
numProcs = inventory.NUM_INDEX_SHARDS + inventory.NUM_DOC_SHARDS + 1
taskID = process.fork_processes(numProcs, max_restarts=0)
port = inventory.BASE_PORT + taskID
if taskID == 0:
app = httpserver.HTTPServer(tornado.web.Application([
(r"/search", Web),
(r"/upload", UploadHandler),
(r"/(.*)", IndexDotHTMLAwareStaticFileHandler, dict(path=SETTINGS['static_path']))
], **SETTINGS))
logging.info("Front end is listening on " + str(port))
else:
if taskID <= inventory.NUM_INDEX_SHARDS:
shardIx = taskID - 1
#data = pickle.load(open("data/index%d.pkl" % (shardIx), "r"))
inverted_path = os.path.join(os.getcwd(),"../assignment5/df_jobs/%d.out" % (shardIx))
logging.info("Inverted file path: %s" % inverted_path)
data = pickle.load(open(inverted_path ,'r'))
idf_path = os.path.join(os.getcwd(), "../assignment5/idf_jobs/0.out")
logIDF = pickle.load(open(idf_path,'r'))
app = httpserver.HTTPServer(web.Application([(r"/index", index.Index, dict(data=data, logIDF=logIDF))]))
logging.info("Index shard %d listening on %d" % (shardIx, port))
else:
shardIx = taskID - inventory.NUM_INDEX_SHARDS - 1
#data = pickle.load(open("data/doc%d.pkl" % (shardIx), "r"))
doc_path = os.path.join(os.getcwd(),"../assignment5/i_df_jobs/%d.out" % (shardIx))
logging.info("Doc Server path %s" % doc_path)
data = pickle.load(open(doc_path, "r"))
app = httpserver.HTTPServer(web.Application([(r"/doc", doc.Doc, dict(data=data))]))
logging.info("Doc shard %d listening on %d" % (shardIx, port))
app.add_sockets(netutil.bind_sockets(port))
IOLoop.current().start()
def main():
entries = {}
for urls_file in urls_files:
with open(urls_file) as f:
for line in f.read().splitlines():
if not line.startswith("#"):
for host in options.hosts.split(","):
entry = Entry.make(line, host.strip(), options.retry_times)
entries[entry.url] = entry
if not entries:
sys.exit(0)
if options.multi_processes != -1:
process.fork_processes(options.multi_processes)
bc = Checker(entries.values(), options.timeout, options.max_clients)
bc.check()
def main():
if options.urls_file:
with open(options.urls_file) as f:
lines = [line for line in f.read().splitlines()
if not line.startswith('#')]
if options.url_template:
lines = [options.url_template % line for line in lines]
requests.extend(lines)
if not requests:
sys.exit(0)
if options.multi_processes != -1:
process.fork_processes(options.multi_processes)
bc = BenchClient(requests, options.timeout, options.max_clients,
options.time_len)
bc.bench()
def main(): taskID = process.fork_processes(7) if taskID == 6: makefrontend() elif taskID<=2: #makeIndexer(taskID) makeDocEnd(taskID) else: makeIndexer(taskID-3) IOLoop.instance().start()
def start(self, num_processes):
'''增加了process_id的返回值。方便识别子进程并初始化不同文件名的logfile等操作。'''
assert not self._started
self._started = True
process_id = 0
if num_processes != 1:
process_id = process.fork_processes(num_processes)
sockets = self._pending_sockets
self._pending_sockets = []
self.add_sockets(sockets)
return process_id
def main():
taskID = process.fork_processes(4)
if taskID == 0:
app = httpserver.HTTPServer(makeFrontend())
app.add_sockets(netutil.bind_sockets(BASE_PORT))
logging.info("Front end is listening on " + str(BASE_PORT))
else:
port = BASE_PORT + taskID
app = httpserver.HTTPServer(makeBackend(port))
app.add_sockets(netutil.bind_sockets(port))
logging.info("Back end %d listening on %d" % (taskID, port))
IOLoop.current().start()
def main():
try:
options.parse_command_line()
port = options.options.port
fork = options.options.fork
setting['PROCESS'] = fork
setting['PORT_GROUP'] = range(port, port + fork)
process.fork_processes(fork, max_restarts=10)
setting['PORT'] = port + process.task_id()
app = web.Application(
handlers=urls,
**SETTINGS
)
app.listen(setting['PORT'], xheaders=True)
loop = ioloop.IOLoop.instance()
loop.start()
except Exception, e:
logger.error(traceback.format_exc(e))
def main():
script_path = os.path.dirname(os.path.realpath(__file__))
config_path = os.path.join(script_path, 'etc/training.cfg')
confirm_path = os.path.join(script_path, 'data/confirmation.text')
config = ConfigParser()
config.read(config_path)
sockets = bind_sockets(config.get('server', 'port'),
config.get('server', 'address'))
fork_processes(config.getint('server', 'instances'))
datastore = DataStore(config.get('database', 'host'),
config.getint('database', 'port'),
config.get('database', 'username'),
config.get('database', 'password'),
config.get('database', 'database'))
with open(confirm_path) as file:
message = file.read()
params_report = {'datastore': datastore,
'api_key': config.get('server', 'api_key')}
params_confirm = {'datastore': datastore,
'message': message}
app = Application([(r"/training/report",
ReportHandler, params_report),
(r"/training/confirm/(.*)",
ConfirmationHandler, params_confirm)])
server = HTTPServer(app,
no_keep_alive=config.get('server', 'no_keep_alive'),
ssl_options={
'certfile': config.get('server', 'certfile'),
'keyfile': config.get('server', 'keyfile')})
server.add_sockets(sockets)
IOLoop.instance().start()
def start(self): """Start up a tornado web server and use the logic method as the request handler""" # find a session and join it self.join_session(self.bot_endpoint) # configure the bot event handler and the user supplied logic bot_server_app = tornado.web.Application([ (r"/event", BotEventHandler, dict( bot_logic=self.bot_logic, player_id=self.player_id, game_id=self.game_id )) ]) # start 'er up sockets = bind_sockets(self.bot_port) print "Bot is listening d[-_-]b" fork_processes(None) server = HTTPServer(bot_server_app) server.add_sockets(sockets) # start listening tornado.ioloop.IOLoop.instance().start()
def cmd_serve(self, *args):
"""
Comando 'serve', responsável por iniciar o servidor de produção.
"""
port = int(args[0]) if args else self.port
ServerLog.info(code='dev_server_starting', args=[port])
app = Application(
routers, cookie_secret=config('SECRET_KEY'),
login_url='/api/v1/usuarios/entrar/',
xsrf_cookies=False)
sockets = netutil.bind_sockets(port)
process.fork_processes(self.fork_processes)
server = HTTPServer(app)
server.add_sockets(sockets)
r.set_loop_type('tornado')
ioloop = IOLoop.current()
ioloop.add_callback(
callback=ServerLog.info,
code='server_started',
args=[self.host, port]
)
ioloop.start()
def start(settings):
application = get_application(settings)
http_server = HTTPServer(application)
unix_socket_enabled = settings.get("unix_socket_enabled")
sockets = []
if unix_socket_enabled:
server_unix_socket_file = settings.get("unix_socket_file")
server_backlog = settings.get("backlog")
# Use unix socket
_logger.info('Bind unix socket file %s', server_unix_socket_file)
socket = netutil.bind_unix_socket(server_unix_socket_file, 0600, server_backlog)
sockets.append(socket)
else:
server_port = settings.get("port")
# Normal way to enable a port for listening the request
_logger.info('Listen on port %d', server_port)
sockets.extend(netutil.bind_sockets(server_port))
process_count = settings.get("process_count")
if not settings.get("debug") and process_count != 1:
if process_count <= 0:
process_count = process.cpu_count()
elif process_count > 1:
_logger.info('Start %d processes', process_count)
process.fork_processes(process_count)
http_server.add_sockets(sockets)
# Start Service
_logger.info('Start tornado server')
try:
IOLoop.instance().start()
except:
_logger.fatal('Start tornado server failed', exc_info = True)
raise
def main():
numProcs = inventory.NUM_WORKERS
taskID = process.fork_processes(numProcs, max_restarts=0)
port = inventory.BASE_PORT + taskID
app = httpserver.HTTPServer(web.Application([
web.url(r"/map", mapper.Map),
web.url(r"/retrieveMapOutput", mapper.RetrieveMapOutput),
web.url(r"/reduce", reducer.Reduce),
web.url(r"/retrieveReduceOutput", reducer.RetrieveReduceOutput),
web.url(r"/coordinator", coordinator.Runner)]))
logging.info("Worker %d listening on %d" % (taskID, port))
app.add_sockets(netutil.bind_sockets(port))
IOLoop.current().start()
def bootstrap( self ):
gLogger.always( "\n === Bootstrapping REST Server === \n" )
ol = ObjectLoader( [ 'DIRAC', 'RESTDIRAC' ] )
result = ol.getObjects( "RESTSystem.API", parentClass = RESTHandler, recurse = True )
if not result[ 'OK' ]:
return result
self.__handlers = result[ 'Value' ]
if not self.__handlers:
return S_ERROR( "No handlers found" )
self.__routes = [ ( self.__handlers[ k ].getRoute(), self.__handlers[k] ) for k in self.__handlers if self.__handlers[ k ].getRoute() ]
gLogger.info( "Routes found:" )
for t in sorted( self.__routes ):
gLogger.info( " - %s : %s" % ( t[0], t[1].__name__ ) )
balancer = RESTConf.balancer()
kw = dict( debug = RESTConf.debug(), log_function = self._logRequest )
if balancer and RESTConf.numProcesses not in ( 0, 1 ):
process.fork_processes( RESTConf.numProcesses(), max_restarts = 0 )
kw[ 'debug' ] = False
if kw[ 'debug' ]:
gLogger.always( "Starting in debug mode" )
self.__app = web.Application( self.__routes, **kw )
port = RESTConf.port()
if balancer:
gLogger.notice( "Configuring REST HTTP service for balancer %s on port %s" % ( balancer, port ) )
self.__sslops = False
else:
gLogger.notice( "Configuring REST HTTPS service on port %s" % port )
self.__sslops = dict( certfile = RESTConf.cert(),
keyfile = RESTConf.key(),
cert_reqs = ssl.CERT_OPTIONAL,
ca_certs = RESTConf.generateCAFile() )
self.__httpSrv = httpserver.HTTPServer( self.__app, ssl_options = self.__sslops )
self.__httpSrv.listen( port )
return S_OK()
def start(self, handlers=None, app_info=None, twork_module=None, **settings):
sockets_list = []
for bind_ip in options.bind_ip.split(","):
if not bind_ip:
break
sockets = tornado.netutil.bind_sockets(options.port, address=bind_ip, backlog=options.backlog)
sockets_list.append(sockets)
if options.num_processes >= 0:
process.fork_processes(options.num_processes)
if twork_module is not None:
twork_module.setup()
if options.timer_start:
CommonTimer().start(twork_module.timer_callback)
self.http_server = tornado.httpserver.HTTPServer(
xheaders=True, request_callback=_TApplication(handlers=handlers, app_info=app_info, **settings)
)
for sockets in sockets_list:
self.http_server.add_sockets(sockets)
def tornado_bidder_run():
"""runs httpapi bidder agent"""
# bind tcp port to launch processes on requests
sockets = netutil.bind_sockets(CONFIG_OBJ["Bidder"]["Port"])
# fork working processes
process.fork_processes(0)
# Tornado app implementation
app = Application([url(r"/", TornadoFixPriceBidAgentRequestHandler)])
# start http servers and attach the web app to it
server = httpserver.HTTPServer(app)
server.add_sockets(sockets)
# perform following actions only in the parent process
process_counter = process.task_id()
if process_counter == 0:
# run dummy ad server
adserver_win = Application([url(r"/", TornadoDummyRequestHandler)])
winport = CONFIG_OBJ["Bidder"]["Win"]
adserver_win.listen(winport)
adserver_evt = Application([url(r"/", TornadoDummyRequestHandler)])
evtport = CONFIG_OBJ["Bidder"]["Event"]
adserver_evt.listen(evtport)
# --instantiate budget pacer
pacer = BudgetControl()
pacer.start(CONFIG_OBJ)
# add periodic event to call pacer
PeriodicCallback(pacer.http_request, CONFIG_OBJ["Banker"]["Period"]).start()
# main io loop. it will loop waiting for requests
IOLoop.instance().start()
def test_multi_process(self):
self.assertFalse(IOLoop.initialized())
port = get_unused_port()
def get_url(path):
return "http://127.0.0.1:%d%s" % (port, path)
sockets = bind_sockets(port, "127.0.0.1")
# ensure that none of these processes live too long
signal.alarm(5) # master process
try:
id = fork_processes(3, max_restarts=3)
except SystemExit, e:
# if we exit cleanly from fork_processes, all the child processes
# finished with status 0
self.assertEqual(e.code, 0)
self.assertTrue(task_id() is None)
for sock in sockets: sock.close()
signal.alarm(0)
return
def main():
numProcs = 3
taskID = process.fork_processes(numProcs, max_restarts=0)
port = BASE_PORT + taskID
if taskID == 0:
app = httpserver.HTTPServer(tornado.web.Application([
(r"/submit", Web)], **SETTINGS))
logging.info("webapp listening on %d" % port)
else:
#load trained model from either dm or dbow
if os.path.isfile(dmLabeled) and os.path.isfile(dbowLabeled):
fname = dmLabeled if taskID == 1 else dbowLabeled
model = Doc2Vec.load(fname)
else:
raise RuntimeError("Must first train doc2vec model")
app = httpserver.HTTPServer(web.Application([(r"/doc2vec", Doc2vecServer, dict(model = model))]))
logging.info("Doc2vec server %d listening on %d" % (taskID, port))
app.add_sockets(netutil.bind_sockets(port))
IOLoop.current().start()
def startup():
define(r'port', 80, int, r'Server listen port')
define(r'service', False, bool, r'Open Scheduled Tasks')
options.parse_command_line()
if(config.Static.Debug):
options.parse_command_line([__file__, r'--service=true', r'--logging=debug'])
settings = {
r'handlers': router.urls,
r'static_path': r'static',
r'template_loader': Jinja2Loader(r'view'),
r'debug': config.Static.Debug,
r'gzip': config.Static.GZip,
r'cookie_secret': config.Static.Secret,
}
sockets = bind_sockets(options.port)
task_id = 0
if(platform.system() == r'Linux'):
task_id = fork_processes(cpu_count())
server = HTTPServer(Application(**settings))
server.add_sockets(sockets)
if(task_id == 0 and options.service):
service.start()
signal.signal(signal.SIGINT, lambda *args : shutdown(server))
signal.signal(signal.SIGTERM, lambda *args : shutdown(server))
app_log.info(r'Startup http server No.{0}'.format(task_id))
IOLoop.instance().start()
def test_multi_process(self):
with ExpectLog(gen_log, "(Starting .* processes|child .* exited|uncaught exception)"):
self.assertFalse(IOLoop.initialized())
sock, port = bind_unused_port()
def get_url(path):
return "http://127.0.0.1:%d%s" % (port, path)
# ensure that none of these processes live too long
signal.alarm(5) # master process
try:
id = fork_processes(3, max_restarts=3)
self.assertTrue(id is not None)
signal.alarm(5) # child processes
except SystemExit, e:
# if we exit cleanly from fork_processes, all the child processes
# finished with status 0
self.assertEqual(e.code, 0)
self.assertTrue(task_id() is None)
sock.close()
return
try:
if id in (0, 1):
self.assertEqual(id, task_id())
server = HTTPServer(self.get_app())
server.add_sockets([sock])
IOLoop.instance().start()
elif id == 2:
self.assertEqual(id, task_id())
sock.close()
# Always use SimpleAsyncHTTPClient here; the curl
# version appears to get confused sometimes if the
# connection gets closed before it's had a chance to
# switch from writing mode to reading mode.
client = HTTPClient(SimpleAsyncHTTPClient)
def fetch(url, fail_ok=False):
try:
return client.fetch(get_url(url))
except HTTPError, e:
if not (fail_ok and e.code == 599):
raise
# Make two processes exit abnormally
fetch("/?exit=2", fail_ok=True)
fetch("/?exit=3", fail_ok=True)
# They've been restarted, so a new fetch will work
int(fetch("/").body)
# Now the same with signals
# Disabled because on the mac a process dying with a signal
# can trigger an "Application exited abnormally; send error
# report to Apple?" prompt.
#fetch("/?signal=%d" % signal.SIGTERM, fail_ok=True)
#fetch("/?signal=%d" % signal.SIGABRT, fail_ok=True)
#int(fetch("/").body)
# Now kill them normally so they won't be restarted
fetch("/?exit=0", fail_ok=True)
# One process left; watch it's pid change
pid = int(fetch("/").body)
fetch("/?exit=4", fail_ok=True)
pid2 = int(fetch("/").body)
self.assertNotEqual(pid, pid2)
# Kill the last one so we shut down cleanly
fetch("/?exit=0", fail_ok=True)
os._exit(0)
except Exception:
logging.error("exception in child process %d", id, exc_info=True)
raise
def main():
from recommendation import recom_worker, recom_front
from searchEngine.backend import back as searchEng_worker
from searchEngine.frontend import front as searchEng_front
from classification.backend import online as classifier
# from recommendation import searchEng_worker, searchEng_front
import mapreduce.framework as framework
from src import color
# from src import tomatoCrawler as TC
C = color.bcolors()
global masterServer, MovieServer, ReviewServer, IdxServer, DocServer, Baseport
print C.HEADER + "=========== Start Crawling ===========" + C.ENDC
# TC.main2Genre()
print C.HEADER + "=========== Find Available Ports ===========" + C.ENDC
getPorts()
print C.OKBLUE + "SuperFront:\t" + str(SuperFront) + C.ENDC
print C.OKBLUE + "masterServer:\t" + str(masterServer) + C.ENDC
print C.OKBLUE + "MovieServer:\t" + str(MovieServer) + C.ENDC
print C.OKBLUE + "ReviewServer:\t" + str(ReviewServer) + C.ENDC
print C.OKBLUE + "IdxServer:\t" + str(IdxServer) + C.ENDC
print C.OKBLUE + "DocServer:\t" + str(DocServer) + C.ENDC
print C.OKBLUE + "ClassifierServer:\t" + str(ClassifierServer) + C.ENDC
print C.HEADER + "=========== Fire Up All Servers ===========" + C.ENDC
uid = fork_processes(NumMaster+NumMovie+NumReview+NumIdx+NumDoc)
if uid == 0:
sockets = bind_sockets(masterServer[uid].split(':')[-1])
myfront = recom_front.FrontEndApp(MovieServer, ReviewServer)
server = myfront.app
elif uid ==1:
sockets = bind_sockets(masterServer[uid].split(':')[-1])
myfront = searchEng_front.FrontEndApp(IdxServer, DocServer)
server = myfront.app
elif uid ==2:
sockets = bind_sockets(masterServer[uid].split(':')[-1])
myClasify = classifier.Application(([(r"/predict?", classifier.PredictionHandler)]))
myClasify.setGenres("./constants/classification_weights/genres.p")
myClasify.setWeights("./constants/classification_weights/big_weight.p")
server = tornado.httpserver.HTTPServer(myClasify )
elif uid < NumMaster + NumMovie:
myIdx = uid - NumMaster
sockets = bind_sockets(MovieServer[myIdx].split(':')[-1])
myback_movie = recom_worker.RecommApp('MovieServer', myIdx, MovieServer[myIdx].split(':')[-1])
server = myback_movie.app
elif uid < NumMaster + NumMovie + NumReview:
myIdx = uid - NumMovie - NumMaster
sockets = bind_sockets(ReviewServer[myIdx].split(':')[-1])
myback_review = recom_worker.RecommApp('ReviewServer', myIdx, ReviewServer[myIdx].split(':')[-1])
server = myback_review.app
elif uid < NumMaster + NumMovie + NumReview + NumIdx:
myIdx = uid-NumMovie-NumReview-NumMaster
sockets = bind_sockets(IdxServer[myIdx].split(':')[-1])
myback_idx = searchEng_worker.BackEndApp('IndexServer', myIdx, IdxServer[myIdx].split(':')[-1])
server = myback_idx.app
elif uid < NumMaster + NumMovie + NumReview + NumIdx + NumDoc:
myIdx = uid-NumMovie-NumReview-NumIdx-NumMaster
sockets = bind_sockets(DocServer[myIdx].split(':')[-1])
myback_doc = searchEng_worker.BackEndApp('DocServer', myIdx, DocServer[myIdx].split(':')[-1])
server = myback_doc.app
server.add_sockets(sockets)
tornado.ioloop.IOLoop.instance().start()
def __enter__(self):
CORE.info('Initialising server process')
if self.__unix:
CORE.info('Using a UNIX socket')
self.__realunixsocket = socket.socket(socket.AF_UNIX,
socket.SOCK_STREAM)
if self.__port:
CORE.info('Using a TCP socket')
try:
self.__realtcpsocket = socket.socket(socket.AF_INET6,
socket.SOCK_STREAM)
except Exception:
CORE.warn(
'Cannot open socket with AF_INET6 (Python reports socket.has_ipv6 is %s), trying AF_INET'
% socket.has_ipv6)
self.__realtcpsocket = socket.socket(socket.AF_INET,
socket.SOCK_STREAM)
for sock in (self.__realtcpsocket, self.__realunixsocket):
if sock is None:
continue
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.setblocking(0)
fd = sock.fileno()
flags = fcntl.fcntl(fd, fcntl.F_GETFD)
fcntl.fcntl(fd, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC)
if self.__ssl and self.__port:
CORE.info('Setting up SSL configuration')
self.crypto_context = SSL.Context(SSL.TLSv1_METHOD)
self.crypto_context.set_cipher_list(
ucr.get('umc/server/ssl/ciphers', 'DEFAULT'))
self.crypto_context.set_options(SSL.OP_NO_SSLv2)
self.crypto_context.set_options(SSL.OP_NO_SSLv3)
self.crypto_context.set_verify(SSL.VERIFY_PEER,
self.__verify_cert_cb)
dir = '/etc/univention/ssl/%s.%s' % (ucr['hostname'],
ucr['domainname'])
try:
self.crypto_context.use_privatekey_file(
os.path.join(dir, 'private.key'))
self.crypto_context.use_certificate_file(
os.path.join(dir, 'cert.pem'))
self.crypto_context.load_verify_locations(
'/etc/univention/ssl/ucsCA/CAcert.pem')
except SSL.Error as exc:
# SSL is not possible
CRYPT.error('Setting up SSL configuration failed: %s' %
(exc, ))
CRYPT.warn('Communication will not be encrypted!')
self.__ssl = False
self.crypto_context = None
self.__realtcpsocket.bind(('', self.__port))
CRYPT.info('Server listening to unencrypted connections')
self.__realtcpsocket.listen(SERVER_MAX_CONNECTIONS)
if self.crypto_context:
self.connection = SSL.Connection(self.crypto_context,
self.__realtcpsocket)
self.connection.setblocking(0)
self.connection.bind(('', self.__port))
self.connection.set_accept_state()
CRYPT.info('Server listening to SSL connections')
self.connection.listen(SERVER_MAX_CONNECTIONS)
elif not self.__ssl and self.__port:
self.crypto_context = None
self.__realtcpsocket.bind(('', self.__port))
CRYPT.info('Server listening to TCP connections')
self.__realtcpsocket.listen(SERVER_MAX_CONNECTIONS)
if self.__unix:
# ensure that the UNIX socket is only accessible by root
old_umask = os.umask(0o077)
try:
self.__realunixsocket.bind(self.__unix)
except EnvironmentError:
if os.path.exists(self.__unix):
os.unlink(self.__unix)
finally:
# restore old umask
os.umask(old_umask)
CRYPT.info('Server listening to UNIX connections')
self.__realunixsocket.listen(SERVER_MAX_CONNECTIONS)
if self.__processes != 1:
self._children = multiprocessing.Manager().dict()
try:
self._child_number = process.fork_processes(
self.__processes, 0)
except RuntimeError as exc:
CORE.warn('Child process died: %s' % (exc, ))
os.kill(os.getpid(), signal.SIGTERM)
raise SystemExit(str(exc))
if self._child_number is not None:
self._children[self._child_number] = os.getpid()
if self.__magic:
self.__bucket = self.__magicClass()
else:
self.signal_new('session_new')
if self.__ssl:
notifier.socket_add(self.connection, self._connection)
if (not self.__ssl and self.__port):
notifier.socket_add(self.__realtcpsocket, self._connection)
if self.__unix:
notifier.socket_add(self.__realunixsocket, self._connection)
return self
futures.append(http.fetch(url))
futures = []
for i in range(5):
url = "%s%d/result" % (BASE_URL, 25800 + i)
logging.info("Fetching %s" % url)
futures.append(http.fetch(url))
responses = yield futures
result = {}
for idx, resp in enumerate(responses):
result[idx] = json.loads(resp.body)
self.write(json.dumps(result))
self.finish()
def makeEnd(index):
port = 25800 + index
data = dataStore()
app = web.Application([
web.url(r"/test", testHandler, dict(data=data)),
web.url(r"/result", getDataHandler, dict(data=data)),
web.url(r"/rrrr", retrieveHandler)
])
app.listen(port)
logging.info("No. %d worker is listening on %d" % (index, port))
if __name__ == "__main__":
taskID = process.fork_processes(5)
makeEnd(taskID)
IOLoop.instance().start()
required=True,
metavar='<PATH_TO_PICKLED_GENRE_LIST>',
help='pickle file of the genres list')
parser.add_argument('--weightsPath',
required=True,
metavar='<PATH_TO_PICKLED_GENRE_WEIGHTS_DICT>',
help='pickle file of the weights dict')
return parser.parse_args(argv)
# python -m classification.servers --genresPath constants/classification_weights/genres.p --weightsPath constants/classification_weights/big_weight.p 3 classification/server_address.json
if __name__ == "__main__":
args = getArguments(sys.argv[1:])
if args.numServer > inventory.NUM_OF_MACHINES:
args.numServer = inventory.NUM_OF_MACHINES
BASE_PORT = 15000
for i in range(args.numServer):
inventory.getWorker('127.0.0.1:{0}'.format(BASE_PORT + 1 + i))
inventory.toJson(args.outputPath)
pid = fork_processes(args.numServer, max_restarts=0)
app = online.Application(([(r"/predict?", online.PredictionHandler)]))
port = BASE_PORT + 1 + pid
app.setGenres(args.genresPath)
app.setWeights(args.weightsPath)
app.listen(port)
print 'A classification server is serving at ' + inventory.MACHINES_IN_USE[
pid]
tornado.ioloop.IOLoop.instance().start()
#!/usr/bin/env python3
from tornado.ioloop import IOLoop
from tornado.process import task_id, fork_processes
from tornado.web import Application, RequestHandler
class MainHandler(RequestHandler):
def get(self):
self.write('Hello, I am ID {}\n'.format(task_id()))
class CrashHandler(RequestHandler):
def get(self):
self.write('Crashing ID {}!\n'.format(task_id()))
IOLoop.current().stop()
if __name__ == '__main__':
app = Application([
(r'/', MainHandler),
(r'/crash', CrashHandler),
])
fork_processes(6)
app.listen(8080 + task_id(), address='127.0.0.1')
IOLoop.current().start()
),
datefmt = "%Y.%m.%d %H:%M:%S",
filename = "log/my_store.log",
filemode = "a"
)
sock_lsn = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
sock_lsn.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock_lsn.setblocking(0)
sock_lsn.bind(("", 11211))
sock_lsn.listen(65535)
while True:
try:
# 该函数在父进程要么exit，要么抛异常，不会返回
task_id = fork_processes(num_processes = 1, max_restarts = 1)
except RuntimeError as e:
logging.error("fork failed: {}".format(e))
time.sleep(60) # 重启间隔
except KeyboardInterrupt as e:
sys.exit(0)
else:
# 子进程
break
io_loop = IOLoop.current()
server = Transformer(io_loop, sock_lsn, 1)
io_loop.add_handler(sock_lsn.fileno(),
server.handle_connection,
io_loop.READ | io_loop.ERROR)
try:
class ECHOServer(TCPServer):
def handle_stream(self,stream,address):
logging.info("echo connecting...%r",address)
ECHO(stream,self)
class Policy(TCPServer):
def handle_stream(self,stream,address):
logging.info("policy ....")
rs = open('socket-policy.xml', 'rt').read()
rs = bytes(rs)
print rs
stream.write(rs)
if __name__ == '__main__':
parse_command_line()
socket1 = netutil.bind_sockets(8000)
#socket2 = netutil.bind_sockets(843)
process.fork_processes(0)
server = ECHOServer()
server.add_sockets(socket1)
#server2 = Policy()
#server2.add_sockets(socket2)
IOLoop.instance().start()
def start(self, app, port, multnum=1):
sockets = bind_sockets(port)
fork_processes(multnum)
server = HTTPServer(app, xheaders=True)
server.add_sockets(sockets)
IOLoop.instance().start()
def watch_dispatch():
#监听更多通道，分散压力
watch_channels = [str(x) for x in xrange(1, 11)]
#兼容
watch_channels.append('dispatch')
while 1:
try:
_, unpacked_msg = dispatch_redis.blpop(watch_channels, 0)
unpacked_msg = msgpack.unpackb(unpacked_msg, encoding='utf-8')
dst_ch = unpacked_msg.get('c')
redis_list = task_ch_redis.get(dst_ch)
if not redis_list:
logger.warn('mis_spell channel: %s' % dst_ch)
continue
randin = randint(0, len(redis_list) - 1)
selected_redis = redis_list[randin]
logger.warn('selected_redis:%s,dst_ch:%s, unpacked_msg:%s' %
(selected_redis, dst_ch, unpacked_msg))
selected_redis.rpush(dst_ch, unpacked_msg.get('p'))
except Exception, ex:
logger.error('blpop fail: {0}'.format(ex), exc_info=True)
if __name__ == '__main__':
logger.init_log('mqd_job', 'mqd_job')
fork_processes(10)
setproctitle('mqd_job')
watch_dispatch()
#The documents are then scored. Each document's score is the inner product (a.k.a. dot product,
#effectively correlation here) of its vector and the query vector. In addition, scores should be
#biased so that documents with the query terms in their title receive especially high scores.
ScoredListing = []
for docID in MyMap:
score = dotProd(MyMap[docID], QueryVector)
ScoredListing.append([score, docID]) ## Bias comes from tfs itself
#Finally, the K highest-scoring documents are written out as a JSON-encoded list of (docID, score) pairs.
ScoredListing.sort(reverse=True) ## check so sorted by score
topPostings = ScoredListing[:10]
TP = []
for posting in topPostings:
TP.append([posting[1], posting[0]])
#print 'Length from ' + str(self._port)
#print len(ScoredListing)
self.write(json.JSONEncoder().encode({"postings": TP}))
if __name__ == "__main__":
task_id = process.fork_processes(inventory.num_ind)
I_BASE_PORT = inventory.port1
#application.listen(I_BASE_PORT)
port = I_BASE_PORT + task_id
app = httpserver.HTTPServer(
web.Application([web.url(r"/index", IndexHandler, dict(port=port))]))
app.add_sockets(netutil.bind_sockets(port))
print('indexServer' + str(task_id) + ' at port: ' + str(port))
tornado.ioloop.IOLoop.current().start()
class StaticPageHandler(RequestHandler):
def initialize(self, page):
self.template = os.path.join(settings.template_path, page + '.html')
def get(self):
self.render(self.template)
application = Application([
url(r'/static/(.*)', StaticFileHandler, {'path': settings.static_path}),
url(r"/?", HomePageHandler, name='home'),
url(r"/m/(.{10})/?", MoodPageHandler, name='mood'),
url(r'/privacy/?', StaticPageHandler, {'page': 'privacy'}),
url(r'/tos/?', StaticPageHandler, {'page': 'tos'}),
],
debug=settings.debug)
if '__main__' == __name__:
if settings.debug:
print('Starting debug server on {}'.format(settings.debug_port))
application.listen(settings.debug_port)
IOLoop.instance().start()
else:
print('Starting production server.')
socket = bind_unix_socket(settings.socket, mode=0o777)
fork_processes(None)
server = HTTPServer(application)
server.add_socket(socket)
IOLoop.instance().start()
def main():
from tornado import options
opts = options.OptionParser()
opts.define("version",
type=bool,
help="show version and exit",
callback=show_version)
opts.define("locators",
default=["localhost:10053"],
type=str,
multiple=True,
help="comma-separated endpoints of locators")
opts.define("cache",
default=DEFAULT_SERVICE_CACHE_COUNT,
type=int,
help="count of instances per service")
opts.define(
"config",
help="path to configuration file",
type=str,
callback=lambda path: opts.parse_config_file(path, final=False))
opts.define("count",
default=1,
type=int,
help="count of tornado processes")
opts.define("port", default=8080, type=int, help="listening port number")
opts.define(
"endpoints",
default=["tcp://localhost:8080"],
type=str,
multiple=True,
help=
"Specify endpoints to bind on: prefix unix:// or tcp:// should be used"
)
opts.define("request_header",
default="X-Request-Id",
type=str,
help="header used as a trace id")
opts.define("forcegen_request_header",
default=False,
type=bool,
help="enable force generation of the request header")
opts.define("sticky_header",
default="X-Cocaine-Sticky",
type=str,
help="sticky header name")
opts.define("gcstats",
default=False,
type=bool,
help="print garbage collector stats to stderr")
# tracing options
opts.define("tracing_chance",
default=DEFAULT_TRACING_CHANCE,
type=float,
help="default chance for an app to be traced")
opts.define("configuration_service",
default="unicorn",
type=str,
help="name of configuration service")
opts.define(
"tracing_conf_path",
default="/zipkin_sampling",
type=str,
help="path to the configuration nodes in the configuration service")
# various logging options
opts.define(
"logging",
default="info",
help=("Set the Python log level. If 'none', tornado won't touch the "
"logging configuration."),
metavar="debug|info|warning|error|none")
opts.define(
"log_to_stderr",
type=bool,
default=None,
help=("Send log output to stderr. "
"By default use stderr if --log_file_prefix is not set and "
"no other logging is configured."))
opts.define("log_file_prefix",
type=str,
default=None,
metavar="PATH",
help=("Path prefix for log file"))
opts.define("datefmt",
type=str,
default="%z %d/%b/%Y:%H:%M:%S",
help="datefmt")
opts.define("generallogfmt",
type=str,
help="log format of general logging system",
default=DEFAULT_GENERAL_LOGFORMAT)
opts.define("accesslogfmt",
type=str,
help="log format of access logging system",
default=DEFAULT_ACCESS_LOGFORMAT)
opts.define("logframework",
type=bool,
default=False,
help="enable logging various framework messages")
opts.define("fingerscrossed",
type=bool,
default=True,
help="enable lazy logging")
# util server
opts.define("utilport",
default=8081,
type=int,
help="listening port number for an util server")
opts.define("utiladdress",
default="127.0.0.1",
type=str,
help="address for an util server")
opts.define("enableutil",
default=False,
type=bool,
help="enable util server")
opts.define("so_reuseport",
default=True,
type=bool,
help="use SO_REUSEPORT option")
opts.parse_command_line()
enable_logging(opts)
logger = logging.getLogger("cocaine.proxy.general")
use_reuseport = False
endpoints = Endpoints(opts.endpoints)
sockets = []
if endpoints.has_unix:
logger.info("Start binding on unix sockets")
for path in endpoints.unix:
logger.info("Binding on %s", path)
sockets.append(bind_unix_socket(path, mode=0o666))
if opts.so_reuseport:
if not support_reuseport():
logger.warning("Your system doesn't support SO_REUSEPORT."
" Bind and fork mechanism will be used")
else:
logger.info("SO_REUSEPORT will be used")
use_reuseport = True
if not use_reuseport and endpoints.has_tcp:
logger.info("Start binding on tcp sockets")
for endpoint in endpoints.tcp:
logger.info("Binding on %s:%d", endpoint.host, endpoint.port)
# We have to bind before fork to distribute sockets to our forks
socks = bind_sockets(endpoint.port, address=endpoint.host)
logger.info(
"Listening %s",
' '.join(str("%s:%s" % s.getsockname()[:2]) for s in socks))
sockets.extend(socks)
if opts.enableutil:
utilsockets = bind_sockets(opts.utilport, address=opts.utiladdress)
logger.info(
"Util server is listening on %s",
' '.join(str("%s:%s" % s.getsockname()[:2]) for s in utilsockets))
try:
if opts.count != 1:
process.fork_processes(opts.count)
if opts.gcstats:
enable_gc_stats()
if use_reuseport and endpoints.has_tcp:
logger.info("Start binding on tcp sockets")
for endpoint in endpoints.tcp:
logger.info("Binding on %s:%d", endpoint.host, endpoint.port)
# We have to bind before fork to distribute sockets to our forks
socks = bind_sockets(endpoint.port,
address=endpoint.host,
reuse_port=True)
logger.info(
"Listening %s", ' '.join(
str("%s:%s" % s.getsockname()[:2]) for s in socks))
sockets.extend(socks)
proxy = CocaineProxy(
locators=opts.locators,
cache=opts.cache,
request_id_header=opts.request_header,
sticky_header=opts.sticky_header,
forcegen_request_header=opts.forcegen_request_header,
default_tracing_chance=opts.tracing_chance)
server = HTTPServer(proxy)
server.add_sockets(sockets)
if opts.enableutil:
utilsrv = HTTPServer(UtilServer(proxy=proxy))
utilsrv.add_sockets(utilsockets)
tornado.ioloop.IOLoop.current().start()
except KeyboardInterrupt:
pass
def main(): taskID = process.fork_processes(MAX_WORKER) makeWorker(taskID) IOLoop.instance().start()
print bcolors.OKGREEN + 'Front end will listen to: ' + bcolors.ENDC + bcolors.OKBLUE + 'http://{}:{}'.format(
socket.gethostname(), Baseport) + bcolors.ENDC + "\n"
# # start indexer and save it properly
# from indexer import indexer
# print bcolors.HEADER + "====== PREPARATION ======" + bcolors.ENDC
# myindexer=indexer.Indexer(Idxservers, Docservers)
# myindexer.genNum()
# myindexer.getCategoryCorrelation('./constants/reducer_tmp_1000lines.txt', Idxservers, Docservers)
# start working
print "\n" + bcolors.HEADER + '====== START FORKING ======' + bcolors.ENDC
from backend import back
from frontend import front
uid = fork_processes(numIdx + numDoc + 1)
if uid == 0:
sockets = bind_sockets(Baseport)
myfront = front.FrontEndApp(Idxservers, Docservers)
server = myfront.app
elif uid < numIdx + 1:
sockets = bind_sockets(Idxservers[uid - 1].split(':')[-1])
myback_idx = back.BackEndApp('indexServer', uid - 1,
Idxservers[uid - 1].split(':')[-1])
server = myback_idx.app
elif uid < numIdx + numDoc + 1:
sockets = bind_sockets(Docservers[uid - numIdx - 1].split(':')[-1])
myback_doc = back.BackEndApp('docServer', uid - numIdx - 1,
Docservers[uid - numIdx - 1].split(':')[-1])
server = myback_doc.app
def main():
try:
options, datafile_json = optparse_lib().parse_args()
#cmdstr="""nohup python easystatserver.py > /dev/null 2>&1 &"""
cmdstr = """nohup ./easystatserver > /dev/null 2>&1 &"""
#status,output=cmd_execute(cmdstr)
import os
os.system(cmdstr)
#print(u"start benchmark test...")
if (options.processnum != -1):
process.fork_processes(options.processnum)
'''logging.basicConfig(level=logging.DEBUG,format='[%(levelname)s] (%(asctime)s) <%(message)s>',datefmt='%a,%Y-%m-%d %H:%M:%S',
filename="./log/process."+str(tornado.process.task_id())+".log",filemode='w')'''
logging.basicConfig(
level=logging.ERROR,
format=
'[%(levelname)s] [%(asctime)s] [%(filename)s-line:%(lineno)d] [%(funcName)s-%(threadName)s] %(message)s',
datefmt='%a,%Y-%m-%d %H:%M:%S',
filename="./log/easyhttpbenchmark.log",
filemode='a')
easyhttpbc = easyhttpbenchmark(options.maxclientnum, options.clientnum,
options.testtime, options.flag,
datafile_json)
easyhttpbc.benchmark_test()
#print(u"benchmark test end...")
except Exception as e:
logging.error(str(e))
try:
from xmlrpclib import ServerProxy
cfg_json = json.load(open("./conf/easyhttpbenchmark.conf", "r"),
encoding='utf-8')
stat_rpc_server = cfg_json['stat_rpc_server']
stat_rpc_port = cfg_json['stat_rpc_port']
svr = ServerProxy("http://" + stat_rpc_server + ":" + stat_rpc_port)
'''print("total_req_cnt:"+str(easyhttpbc.total_req_cnt))
print("total_res_cnt:"+str(easyhttpbc.total_res_cnt))
print("total_err_cnt:"+str(easyhttpbc.total_err_cnt))
print("total_nul_cnt:"+str(easyhttpbc.total_nul_cnt))'''
import multiprocessing
cpu_count = multiprocessing.cpu_count()
if (options.processnum != 0):
svr.stat_maxclientnum(options.processnum * options.maxclientnum)
svr.stat_clientnum(options.processnum * options.clientnum)
else:
svr.stat_maxclientnum(cpu_count * options.maxclientnum)
svr.stat_clientnum(cpu_count * options.clientnum)
svr.set_test_time(easyhttpbc.testtime)
svr.stat_total_req_cnt(easyhttpbc.total_req_cnt)
svr.stat_total_res_cnt(easyhttpbc.total_res_cnt)
svr.stat_total_err_cnt(easyhttpbc.total_err_cnt)
svr.stat_total_nul_cnt(easyhttpbc.total_nul_cnt)
svr.stat_total_below_10(easyhttpbc.below_10)
svr.stat_total_between_10_20(easyhttpbc.between_10_20)
svr.stat_total_between_20_30(easyhttpbc.between_20_30)
svr.stat_total_over_30(easyhttpbc.over_30)
svr.stat_total_res_time(easyhttpbc.total_res_time)
except Exception as e:
logging.error(str(e))
define('num_processes', default=4, help='sub-processes count', type=int)
if __name__ == '__main__':
os.chdir(os.path.dirname(os.path.realpath(__file__)))
opt.parse_command_line()
routes = c.handlers + c.views
app = Application(routes,
default_handler_class=c.InvalidPageHandler,
ui_modules=c.modules,
xsrf_cookies=True)
try:
ssl_options = not opt.debug and app.site.get('https') or None
server = HTTPServer(app, xheaders=True, ssl_options=ssl_options)
sockets = netutil.bind_sockets(opt.port, family=socket.AF_INET)
fork_id = 0 if opt.debug or os.name == 'nt' else process.fork_processes(
opt.num_processes)
server.add_sockets(sockets)
protocol = 'https' if ssl_options else 'http'
logging.info('Start the service #%d v%s on %s://localhost:%d' %
(fork_id, app.version, protocol, opt.port))
if fork_id == 0 and app.db and app.db.page.count_documents(
{}) == 0 and os.path.exists(app.config.get('data_path', '')):
script = 'python3 utils/add_pages.py --uri={0} --db_name={1} --json_path={2}/json'.format(
app.db_uri, app.config['database']['name'],
app.config['data_path'])
os.system(script)
os.system('ln -s %s static/ocr_img' % app.config['data_path'])
ioloop.IOLoop.current().start()
except KeyboardInterrupt:
app.stop()
#print (url)
response = yield http_client.fetch(url)
listText = response.body[14:-3]
resp = json.loads(response.body.decode())
DocList = resp['results'][0]
if (DocList['found']):
results.append(DocList)
self.finish(json.JSONEncoder().encode({"num_results": len(results), "results": results}))
#print 'Results: '
print (str(len(results)) + ' results found! in ' + str(loops) + ' loops')
if __name__ == "__main__":
task_id = process.fork_processes(3)
if(task_id ==0):
application = tornado.web.Application([
(r"/search", FrontEndHandler), (r"/person", actorHandler)
], **SETTINGS)
MAX_PORT = 49152
MIN_PORT = 10000
BASE_PORT = int(hashlib.md5(getpass.getuser().encode()).hexdigest()[:8], 16) % \
(MAX_PORT - MIN_PORT) + MIN_PORT
print ('FrontEnd at port: ' + str(BASE_PORT))
print ('Host name: ' + str(socket.gethostname()))
application.listen(BASE_PORT)
elif task_id == 1:
os.system('python3.6 -m assignment2.indexServer')
else:
from tornado import ioloop, netutil, process
from tornado.httpserver import HTTPServer
from tornado.web import Application
from tornado.options import define, options
import socket
from controller.page import MainHandler, PageHandler
define('port', default=8001, help='run port', type=int)
define('debug', default=True, help='the debug mode', type=bool)
define('num_processes', default=4, help='sub-processes count', type=int)
if __name__ == '__main__':
options.parse_command_line()
options.debug = options.debug and options.port != 80
handlers = [(r'/', MainHandler), (r'/(\w+)', PageHandler)]
app = Application(
handlers,
debug=options.debug,
cookie_secret='R1sl9JqfQnCOS+aAR0fPVPpw5LzQOkzKudChgWnbhKw=',
static_path='static',
template_path='views')
server = HTTPServer(app, xheaders=True)
sockets = netutil.bind_sockets(options.port, family=socket.AF_INET)
fork_id = 0 if options.debug else process.fork_processes(
options.num_processes)
server.add_sockets(sockets)
print('Start http://localhost:%d' % (options.port, ))
ioloop.IOLoop.current().start()
def get(self):
time.sleep(1) # simulate longer query
cur = connect(db='tornado', user='******').cursor(cursors.DictCursor)
cur.execute("SELECT * FROM foo")
self.write(json.dumps(list(cur.fetchall())))
class FrontendHandler(web.RequestHandler):
@gen.coroutine
def get(self):
http_client = httpclient.AsyncHTTPClient(max_clients=500)
response = yield http_client.fetch("http://localhost:8001/foo")
self.set_header("Content-Type", 'application/json')
self.write(response.body)
if __name__ == "__main__":
number_of_be_tasks = int(sys.argv[1]) if len(sys.argv) > 1 else 20
number_of_fe_tasks = int(sys.argv[2]) if len(sys.argv) > 2 else 1
fe_sockets = netutil.bind_sockets(8000) # need to bind sockets
be_sockets = netutil.bind_sockets(8001) # before forking
task_id = process.fork_processes(number_of_be_tasks + number_of_fe_tasks)
if task_id < number_of_fe_tasks:
handler_class = FrontendHandler
sockets = fe_sockets
else:
handler_class = BackendHandler
sockets = be_sockets
httpserver.HTTPServer(web.Application([(r"/foo", handler_class)])
).add_sockets(sockets)
ioloop.IOLoop.instance().start()
# tornado
from tornado.process import fork_processes
import tornado.ioloop
import tornado.web
# package
from config import inventory
from backend import map, reduce, application
if __name__ == "__main__":
BASE_PORT = 20000
path = 'mapreduce_workers.json'
for i in range(inventory.NUM_OF_MACHINES):
inventory.getWorker('127.0.0.1:{0}'.format(BASE_PORT + 1 + i))
inventory.toJson(path)
pid = fork_processes(inventory.NUM_OF_MACHINES, max_restarts=0)
app = application.Application(([
(r"/map?", map.MapHandler),
(r"/retrieveMapOutput?", map.RetrieveOutputHandler),
(r"/reduce?", reduce.ReduceHandler),
]))
port = BASE_PORT + 1 + pid
app.setInventory(path)
app.listen(port)
print 'A worker is working at ' + inventory.MACHINES_IN_USE[pid]
tornado.ioloop.IOLoop.instance().start()
def test_multi_process(self):
self.assertFalse(IOLoop.initialized())
port = get_unused_port()
def get_url(path):
return "http://127.0.0.1:%d%s" % (port, path)
sockets = bind_sockets(port, "127.0.0.1")
# ensure that none of these processes live too long
signal.alarm(5) # master process
id = fork_processes(3, max_restarts=3)
if id is None:
# back in the master process; everything worked!
self.assertTrue(task_id() is None)
for sock in sockets:
sock.close()
signal.alarm(0)
return
signal.alarm(5) # child process
try:
if id in (0, 1):
signal.alarm(5)
self.assertEqual(id, task_id())
server = HTTPServer(self.get_app())
server.add_sockets(sockets)
IOLoop.instance().start()
elif id == 2:
signal.alarm(5)
self.assertEqual(id, task_id())
for sock in sockets:
sock.close()
client = HTTPClient()
def fetch(url, fail_ok=False):
try:
return client.fetch(get_url(url))
except HTTPError, e:
if not (fail_ok and e.code == 599):
raise
# Make two processes exit abnormally
fetch("/?exit=2", fail_ok=True)
fetch("/?exit=3", fail_ok=True)
# They've been restarted, so a new fetch will work
int(fetch("/").body)
# Now the same with signals
# Disabled because on the mac a process dying with a signal
# can trigger an "Application exited abnormally; send error
# report to Apple?" prompt.
#fetch("/?signal=%d" % signal.SIGTERM, fail_ok=True)
#fetch("/?signal=%d" % signal.SIGABRT, fail_ok=True)
#int(fetch("/").body)
# Now kill them normally so they won't be restarted
fetch("/?exit=0", fail_ok=True)
# One process left; watch it's pid change
pid = int(fetch("/").body)
fetch("/?exit=4", fail_ok=True)
pid2 = int(fetch("/").body)
self.assertNotEqual(pid, pid2)
# Kill the last one so we shut down cleanly
fetch("/?exit=0", fail_ok=True)
os._exit(0)
except Exception:
logging.error("exception in child process %d", id, exc_info=True)
raise
def main():
from tornado import options
default_general_logformat = "[%(asctime)s.%(msecs)d]\t[%(filename).5s:%(lineno)d]\t%(levelname)s\t%(message)s"
default_access_logformat = "[%(asctime)s.%(msecs)d]\t[%(filename).5s:%(lineno)d]\t%(levelname)s\t%(trace_id)s\t%(message)s"
opts = options.OptionParser()
opts.define("version",
type=bool,
help="show version and exit",
callback=show_version)
opts.define("locators",
default=["localhost:10053"],
type=str,
multiple=True,
help="comma-separated endpoints of locators")
opts.define("cache",
default=DEFAULT_SERVICE_CACHE_COUNT,
type=int,
help="count of instances per service")
opts.define(
"config",
help="path to configuration file",
type=str,
callback=lambda path: opts.parse_config_file(path, final=False))
opts.define("count",
default=1,
type=int,
help="count of tornado processes")
opts.define(
"endpoints",
default=["tcp://localhost:8080"],
type=str,
multiple=True,
help=
"Specify endpoints to bind on: prefix unix:// or tcp:// should be used"
)
opts.define("request_header",
default="X-Request-Id",
type=str,
help="header used as a trace id")
opts.define("forcegen_request_header",
default=False,
type=bool,
help="enable force generation of the request header")
opts.define("sticky_header",
default="X-Cocaine-Sticky",
type=str,
help="sticky header name")
opts.define("gcstats",
default=False,
type=bool,
help="print garbage collector stats to stderr")
opts.define("srwconfig", default="", type=str, help="path to srwconfig")
opts.define("allow_json_rpc",
default=True,
type=bool,
help="allow JSON RPC module")
opts.define("mapped_headers",
default=[],
type=str,
multiple=True,
help="pass specified headers as cocaine headers")
# tracing options
opts.define("tracing_chance",
default=DEFAULT_TRACING_CHANCE,
type=float,
help="default chance for an app to be traced")
opts.define("configuration_service",
default="unicorn",
type=str,
help="name of configuration service")
opts.define(
"tracing_conf_path",
default="/zipkin_sampling",
type=str,
help="path to the configuration nodes in the configuration service")
# various logging options
opts.define(
"logging",
default="info",
help=("Set the Python log level. If 'none', tornado won't touch the "
"logging configuration."),
metavar="debug|info|warning|error|none")
opts.define("log_to_cocaine",
default=False,
type=bool,
help="log to cocaine")
opts.define(
"log_to_stderr",
type=bool,
default=None,
help=("Send log output to stderr. "
"By default use stderr if --log_file_prefix is not set and "
"no other logging is configured."))
opts.define("log_file_prefix",
type=str,
default=None,
metavar="PATH",
help=("Path prefix for log file"))
opts.define("datefmt",
type=str,
default="%z %d/%b/%Y:%H:%M:%S",
help="datefmt")
opts.define("generallogfmt",
type=str,
help="log format of general logging system",
default=default_general_logformat)
opts.define("accesslogfmt",
type=str,
help="log format of access logging system",
default=default_access_logformat)
opts.define("logframework",
type=bool,
default=False,
help="enable logging various framework messages")
# util server
opts.define("utilport",
default=8081,
type=int,
help="listening port number for an util server")
opts.define("utiladdress",
default="127.0.0.1",
type=str,
help="address for an util server")
opts.define("enableutil",
default=False,
type=bool,
help="enable util server")
opts.define("client_id",
default=0,
type=int,
help="client id used for authentication")
opts.define("client_secret",
default='',
type=str,
help="client secret used for authentication")
opts.parse_command_line()
srw_config = None
if opts.srwconfig:
try:
srw_config = load_srw_config(opts.srwconfig)
except Exception as err:
print("unable to load SRW config: %s" % err)
exit(1)
use_reuseport = hasattr(socket, "SO_REUSEPORT")
endpoints = Endpoints(opts.endpoints)
sockets = []
for path in endpoints.unix:
sockets.append(bind_unix_socket(path, mode=0o666))
if not use_reuseport:
for endpoint in endpoints.tcp:
# We have to bind before fork to distribute sockets to our forks
socks = bind_sockets(endpoint.port, address=endpoint.host)
sockets.extend(socks)
if opts.enableutil:
utilsockets = bind_sockets(opts.utilport, address=opts.utiladdress)
try:
if opts.count != 1:
process.fork_processes(opts.count)
enable_logging(opts)
if opts.gcstats:
enable_gc_stats()
if use_reuseport:
for endpoint in endpoints.tcp:
# We have to bind before fork to distribute sockets to our forks
socks = bind_sockets(endpoint.port,
address=endpoint.host,
reuse_port=True)
sockets.extend(socks)
proxy = CocaineProxy(
locators=opts.locators,
cache=opts.cache,
request_id_header=opts.request_header,
sticky_header=opts.sticky_header,
forcegen_request_header=opts.forcegen_request_header,
default_tracing_chance=opts.tracing_chance,
srw_config=srw_config,
allow_json_rpc=opts.allow_json_rpc,
client_id=opts.client_id,
client_secret=opts.client_secret,
mapped_headers=opts.mapped_headers)
server = HTTPServer(proxy)
server.add_sockets(sockets)
if opts.enableutil:
utilsrv = HTTPServer(UtilServer(proxy=proxy))
utilsrv.add_sockets(utilsockets)
tornado.ioloop.IOLoop.current().start()
except KeyboardInterrupt:
pass
def main():
parser = argparse.ArgumentParser(
'SEASaw - A Search Engine For Video Content')
parser.add_argument("--gca_credentials_path",
action="store",
default=None,
dest="gca_credentials_path")
parser.add_argument("--database_password",
action="store",
default=None,
dest="database_password")
parser.add_argument("--imgur_password",
action="store",
default=None,
dest="imgur_password")
parser.add_argument(
"-s", action="store_true",
dest="run_scraper") # will not run on linux box, due to dependencies
parser.add_argument("-l", action="store_true",
dest="local") # local env and linserv have differences
args = parser.parse_args()
if args.local:
inventory.set_local()
else:
inventory.set_linserv()
if (args.gca_credentials_path is None) or (args.database_password is None):
print(
"start - Missing credential path or database password, datastore will not be loaded"
)
else:
proxy.start(args.gca_credentials_path)
dao.init(args.database_password)
# spin up component APIs
process_id = process.fork_processes(len(inventory.ports), max_restarts=100)
if process_id is 0:
if args.run_scraper:
print("start - initiating scraper")
# youtube scraper
for i in range(0, 5):
term_index = randint(0, len(inventory.search_terms) - 1)
term = inventory.search_terms[term_index]
print("start - setting scraper to find 10 results for " + term)
scraper.start(term, 10)
print("start - resting scraper")
time.sleep(3600)
print("start - scraper finished")
elif process_id is 1:
# imgur uploader
if args.imgur_password is None or args.run_scraper is False:
print("start - imgur uploader not running")
else:
print("start - imgur uploader running")
while True:
result = imguruploader.start(args.imgur_password)
if result is 1:
print(
"start - rate limit exceeded, imgur component will pause for a while"
)
time.sleep(120)
elif result is 2:
print(
"start - nothing for imgur uploader to do, imgur component will pause for a while"
)
time.sleep(120)
elif process_id is 2:
# database uploader
if args.run_scraper:
print("start - initiating database uploader")
while True:
datasourceuploader.start()
time.sleep(30)
elif process_id is 3 or process_id is 4:
# datasource api
if args.local:
instance = Application([(r"/healthcheck", HealthCheckHandler),
(r"/results/(.*)", ResultGetterHandler),
(r"/results", ResultQueryHandler),
(r"/(.*)", StaticFileHandler, {
"path":
"static/apidocs/datasource_local/",
"default_filename": "index.html"
})])
else:
instance = Application([(r"/healthcheck", HealthCheckHandler),
(r"/results/(.*)", ResultGetterHandler),
(r"/results", ResultQueryHandler),
(r"/(.*)", StaticFileHandler, {
"path": "static/apidocs/datasource/",
"default_filename": "index.html"
})])
port = inventory.ports[process_id - 3]
instance.listen(port)
print("start - Data Source Interface listening on port " + str(port))
else:
# Index server threads
port = inventory.ports[process_id]
if (port == 25285):
if (args.local): #vagrant
instance = Application(
[(r'/search', SearchHandler),
(r'/(.*)', IndexDotHTMLAwareStaticFileHandler,
dict(path=SETTINGS['static_path']))], **SETTINGS)
print("start - Frontend Interface listening on port " +
str(port))
instance.listen(port)
else: #linserv
instance = Application(
[(r'/search', SearchHandlerL),
(r'/(.*)', IndexDotHTMLAwareStaticFileHandlerL,
dict(path=SETTINGS['static_path']))], **SETTINGS)
print("start - Frontend Interface listening on port " +
str(port))
instance.listen(port)
IOLoop.current().start()
class ReqHandler(RequestHandler):
def initialize(self, db):
self.db = db
def get(self, query_string):
r = self.db.search(query_string)
self.write({'size': len(r), 'entries': r})
def make_app(db):
return Application([('/s/(.+)', ReqHandler, dict(db=db))])
def load_config(config_file="config.yml"):
with open(config_file, 'r') as config:
return yaml.load(config)
if __name__ == '__main__':
cfg = load_config()
db = init_db(cfg['database']['DATA_SOURCE'])
app = make_app(db)
sockets = bind_sockets(cfg['server']['PORT'])
fork_processes(0)
server = HTTPServer(app)
server.add_sockets(sockets)
logging.info("Server are listening in %d port ", cfg['server']['PORT'])
IOLoop.current().start()
@coroutine
def execute(self, fut):
conn = pymysql.connect(**CONFIG_CONNECT)
cursor = conn.cursor()
cursor.execute('select * from lbe_manager.ec2_and_user')
try:
result = yield cursor.fetchall()
except:
result = []
fut.set_result(json.dumps(result))
@coroutine
def get(self, *args, **kwargs):
future = yield self.new_handle(self.execute)
print('-' * 10, future)
self.finish('hello world')
def post(self, *args, **kwargs):
pass
if __name__ == '__main__':
app = Application([(r'/handle', BaseHandle)])
sockets = netutil.bind_sockets(6667, address='0.0.0.0', reuse_port=True)
process.fork_processes(os.cpu_count())
server = httpserver.HTTPServer(app)
server.add_sockets(sockets)
print('测试开始')
ioloop.IOLoop.current().start()
