def run_in_executor( self, executor: Optional[concurrent.futures.Executor], func: Callable[..., _T], *args: Any ) -> Awaitable[_T]: """Runs a function in a ``concurrent.futures.Executor``. If ``executor`` is ``None``, the IO loop's default executor will be used. Use `functools.partial` to pass keyword arguments to ``func``. .. versionadded:: 5.0 """ if executor is None: if not hasattr(self, "_executor"): from tornado.process import cpu_count self._executor = concurrent.futures.ThreadPoolExecutor( max_workers=(cpu_count() * 5) ) # type: concurrent.futures.Executor executor = self._executor c_future = executor.submit(func, *args) # Concurrent Futures are not usable with await. Wrap this in a # Tornado Future instead, using self.add_future for thread-safety. t_future = Future() # type: Future[_T] self.add_future(c_future, lambda f: chain_future(f, t_future)) return t_future
def __init__(self): self._is_linux = (platform.system() == r'Linux') # 判定当前平台 self._process_id = 0 self._process_num = cpu_count() # cup数 self._sockets = bind_sockets(Config.Port) # 创建socket对象,绑定端口 if self._is_linux: # 如果是linux系统 self._process_id = fork_processes(self._process_num) # self._init_options() AsyncIOMainLoop().install() self._event_loop = asyncio.get_event_loop() self._server = HTTPServer(Application(**self._settings)) signal.signal(signal.SIGINT, self.stop) signal.signal(signal.SIGTERM, self.stop) from model.base import initialize as model_initialize IOLoop.instance().run_sync(model_initialize)
def main(): ''' 读取配置,启动服务 ''' config = ConfigParser() config.read('config.cfg') log_path = config.get("default", "logpath") num_processes = int(config.get("default", "num_processes")) port = int(config.get("default", "port")) # 初始化日志服务 initLog(log_path) # 服务最大线程不超过cpu核数 cpu_count=process.cpu_count() if num_processes>cpu_count: print("This server is only "+str(cpu_count)+" cores, input value is too large.") num_processes = 0 # 启动服务 app = App(config) http_server = httpserver.HTTPServer(app) http_server.bind(port) http_server.start(num_processes) tornado.ioloop.IOLoop.instance().start()
def __init__(self, *args, **kwargs): super(WebHandler, self).__init__(*args, **kwargs) self.io_loop = IOLoop.current() self.executor = ThreadPoolExecutor(cpu_count(), thread_name_prefix='WEBSERVER-' + self.__class__.__name__.upper())
def run(custom_server_settings=None): # redefine tornado default log level (if it not set so will be used) options.logging = None if settings.RUN_IN_BACKGROUND else settings.LOG_LEVEL # get options from cmd parse_command_line() # create log file if needed if settings.RUN_IN_BACKGROUND: Application.setup_logging() # create http server Application.SERVER_INSTANCE = httpserver.HTTPServer(Application(custom_server_settings)) if settings.RUN_IN_BACKGROUND: Application.create_pid_file() Application.SERVER_INSTANCE.bind(options.port, options.host) cpu_count = process.cpu_count() Application.SERVER_INSTANCE.start(4 if cpu_count < 4 else cpu_count) # forking else: Application.SERVER_INSTANCE.listen(options.port, options.host) # graceful shutdown signal.signal(signal.SIGTERM, lambda sig, frame: ioloop.IOLoop.instance().add_callback(Application.shutdown)) signal.signal(signal.SIGINT, lambda sig, frame: ioloop.IOLoop.instance().add_callback(Application.shutdown)) ioloop.IOLoop.instance().start()
def run(self): start_url = target_url + blog_path yield self._fetch_blog_list_page(start_url) for _ in xrange(cpu_count()): self._fetch_essay_content() yield self._q.join()
def start_stats(root_topic="snowpear"): context = zmq.Context(cpu_count()) pub = context.socket(zmq.PUB) pub.bind(options.stats_endpoint) handler = PUBHandler(pub) handler.root_topic = root_topic logger = logging.getLogger() logger.addHandler(handler)
def __init__(self, **kwargs): kwargs["cfg"] = { "mysql": { 'host':'localhost', 'database':'mysql', 'user':'******', 'password':'******' }, "redis": { 'max_connections': cpu_count() } } kwargs["redis"] = tornadoredis.ConnectionPool(wait_for_available = True, max_connections = kwargs["cfg"]["redis"]["max_connections"]) handlers = [(r"/", IndexHandler, kwargs), (r"/ls", CmdHandler)] tornado.web.Application.__init__(self, handlers)
class WebHandler(BaseHandler): def __init__(self, *args, **kwargs): super(WebHandler, self).__init__(*args, **kwargs) self.io_loop = IOLoop.current() executor = ThreadPoolExecutor(cpu_count()) @authenticated @coroutine def get(self, route, *args, **kwargs): try: # route -> method obj route = route.strip('/').replace('.', '_').replace('-', '_') or 'index' method = getattr(self, route) results = yield self.async_call(method) self.finish(results) except AttributeError: logger.log( 'Failed doing webui request "{0}": {1}'.format( route, traceback.format_exc()), logger.DEBUG) raise HTTPError(404) @run_on_executor def async_call(self, function): try: kwargs = self.request.arguments for arg, value in six.iteritems(kwargs): if len(value) == 1: kwargs[arg] = xhtml_escape(value[0]) elif isinstance(value, six.string_types): kwargs[arg] = xhtml_escape(value) elif isinstance(value, list): kwargs[arg] = [xhtml_escape(v) for v in value] else: raise Exception result = function(**kwargs) return result except OSError as e: return Template( "Looks like we do not have enough disk space to render the page! {error}" ).render_unicode(data=e.message) except Exception: logger.log( 'Failed doing webui callback: {0}'.format( (traceback.format_exc())), logger.ERROR) raise # post uses get method post = get
def _initialize(self, **kwargs): Utils.log.info( f'{package_slogan}\nhagworm version {package_version}\npython version {sys.version}' ) self._process_num = kwargs.get(r'process_num', 1) self._async_initialize = kwargs.get(r'async_initialize', None) self._background_service = kwargs.get(r'background_service', None) self._background_process = kwargs.get(r'background_process', None) self._process_id = 0 self._process_num = self._process_num if self._process_num > 0 else cpu_count( ) if self._background_service is None: pass elif not isinstance(self._background_service, TaskInterface): raise TypeError( r'Background Service Dot Implemented Task Interface') if self._background_process is None: pass elif isinstance(self._background_process, TaskInterface): self._process_num += 1 else: raise TypeError( r'Background Process Dot Implemented Task Interface') log_level = kwargs.get(r'log_level', r'info').upper() log_file_path = kwargs.get(r'log_file_path', None) if log_file_path: Utils.log.remove() log_file_path = Utils.path.join(kwargs[r'log_file_path'], r'runtime_{time}.log') Utils.log.add(sink=log_file_path, level=log_level, enqueue=True, backtrace=kwargs.get(r'debug', False), rotation=r'00:00', retention=kwargs.get(r'log_file_num_backups', 7)) else: Utils.log.level(log_level) logging.getLogger(None).addHandler(_InterceptHandler())
class Config(object): cfg_path = '/etc/ansible/api.cfg' host = '127.0.0.1' port = 8765 sign_key = 'YOUR_SIGNATURE_KEY_HERE' log_path = '/var/log/ansible-api.log' allow_ip = [] thread_pool_size = cpu_count( ) * 2 # adapt the number of thread pool size to the number of cpu cores dir_script = '' dir_playbook = '' def __init__(self): cf = ConfigParser.ConfigParser() cf.read(Config.cfg_path) try: cf.options('default') except: pass else: if cf.has_option('default', 'host'): self.host = cf.get('default', 'host') if cf.has_option('default', 'port'): self.port = cf.get('default', 'port') if cf.has_option('default', 'sign_key'): self.sign_key = cf.get('default', 'sign_key') if cf.has_option('default', 'log_path'): self.log_path = cf.get('default', 'log_path') if cf.has_option('default', 'allow_ip'): self.allow_ip = cf.get('default', 'allow_ip').split() if cf.has_option('default', 'thread_pool_size'): self.thread_pool_size = cf.get('default', 'thread_pool_size') try: cf.options('directory') except: pass else: if cf.has_option('directory', 'script'): self.dir_script = cf.get('directory', 'script') if cf.has_option('directory', 'playbook'): self.dir_playbook = cf.get('directory', 'playbook') @staticmethod def Get(attr): cfg = Config() return getattr(cfg, attr, '')
def __init__(self, **kwargs): self._debug = kwargs.get(r'debug', False) self._process_num = kwargs.get(r'process_num', 1) self._async_initialize = kwargs.get(r'async_initialize', None) self._background_service = kwargs.get(r'background_service', None) self._background_process = kwargs.get(r'background_process', None) self._process_id = 0 self._process_num = self._process_num if self._process_num > 0 else cpu_count( ) # 后台服务任务对象 if self._background_service is None: pass elif not isinstance(self._background_service, TaskInterface): raise TypeError( r'Background Service Dot Implemented Task Interface') # 服务进程任务对象,服务进程不监听端口 if self._background_process is None: pass elif isinstance(self._background_process, TaskInterface): self._process_num += 1 else: raise TypeError( r'Background Process Dot Implemented Task Interface') self._init_logger( kwargs.get(r'log_level', r'info').upper(), kwargs.get(r'log_handler', None), kwargs.get(r'log_file_path', None), kwargs.get(r'log_file_rotation', DEFAULT_LOG_FILE_ROTATOR), kwargs.get(r'log_file_retention', 0xff)) environment = Utils.environment() Utils.log.info(f'{package_slogan}' f'hagworm {package_version}\n' f'python {environment["python"]}\n' f'system {" ".join(environment["system"])}') install_uvloop() self._event_loop = None
def __init__(self, **kwargs): kwargs["cfg"] = { "mysql": { 'host': 'localhost', 'database': 'mysql', 'user': '******', 'password': '******' }, "redis": { 'max_connections': cpu_count() } } kwargs["redis"] = tornadoredis.ConnectionPool( wait_for_available=True, max_connections=kwargs["cfg"]["redis"]["max_connections"]) handlers = [(r"/", IndexHandler, kwargs), (r"/ls", CmdHandler)] tornado.web.Application.__init__(self, handlers)
class WebHandler(BaseHandler): """ Base Handler for the web server """ def __init__(self, *args, **kwargs): super(WebHandler, self).__init__(*args, **kwargs) self.io_loop = IOLoop.current() executor = ThreadPoolExecutor(cpu_count()) @authenticated @coroutine def get(self, route, *args, **kwargs): try: # route -> method obj route = route.strip('/').replace('.', '_') or 'index' method = getattr(self, route) results = yield self.async_call(method) self.finish(results) except Exception: logger.log( u'Failed doing web ui request {route!r}: {error}'.format( route=route, error=traceback.format_exc()), logger.DEBUG) raise HTTPError(404) @run_on_executor def async_call(self, function): try: kwargs = self.request.arguments for arg, value in iteritems(kwargs): if len(value) == 1: kwargs[arg] = value[0] result = function(**kwargs) return result except Exception: logger.log( u'Failed doing web ui callback: {error}'.format( error=traceback.format_exc()), logger.ERROR) raise # post uses get method post = get
def __init__(self, entries, timeout, max_clients): assert entries task_id = process.task_id() if options.multi_processes == -1: process_num = 1 elif options.multi_processes == 0: process_num = process.cpu_count() else: process_num = options.multi_processes self._io_loop = ioloop.IOLoop() self._client = httpclient.AsyncHTTPClient(self._io_loop, max_clients=max_clients) self.timeout = timeout self.max_clients = max_clients self.requests = dict([(self.get_request(e), e) for e in entries]) self.partial_requests = self.requests.keys()[task_id::process_num] self.count = len(self.partial_requests)
def output_sys_info(): """ 输出服务器相关配置信息 :param kwargs: :return: """ tbl_out = tabulate([ ['sys_time', timetool.get_current_time_string()], ['dtlib', dtlib.VERSION], ['py_ver', platform.python_version()], ['server_ver', tornado.version], ['cpu_count', process.cpu_count()], ], tablefmt='grid') print(tbl_out) # 启动前输出系统信息,在日志中有所体现---Ubuntu系统 os.system('uname -a') os.system('ifconfig|grep addr')
def __init__(self, *args, **kwargs): kwargs["async"] = True if "thread_pool" in kwargs: self.__thread_pool = kwargs.pop("thread_pool") else: self.__thread_pool = futures.ThreadPoolExecutor(cpu_count()) self.__connection = connect(*args, **kwargs) self.__io_loop = IOLoop.current() self.__connected = False log.debug("Trying to connect to postgresql") f = self.__wait() self.__io_loop.add_future(f, self.__on_connect) self.__queue = Queue() self.__has_active_cursor = False for method in ("get_backend_pid", "get_parameter_status"): setattr(self, method, self.__futurize(method))
def __init__(self, *args, **kwargs): kwargs['async'] = True if "thread_pool" in kwargs: self.__thread_pool = kwargs.pop('thread_pool') else: self.__thread_pool = futures.ThreadPoolExecutor(cpu_count()) self.__connection = connect(*args, **kwargs) self.__io_loop = IOLoop.current() self.__connected = False log.debug("Trying to connect to postgresql") f = self.__wait() self.__io_loop.add_future(f, self.__on_connect) self.__queue = Queue() self.__has_active_cursor = False for method in ('get_backend_pid', 'get_parameter_status'): setattr(self, method, self.__futurize(method))
def main(): if not len(log_files): print 'please specified log files.' sys.exit(0) process_num = { -1: 1, 0: process.cpu_count(), }.get(options.multi_processes, options.multi_processes) la = LogAnalyzer(options.max_clients, process_num) for log_file in log_files: with open(log_file) as f: for line in f.read().splitlines(): la.append(LogEntry.make(line)) la.run() la.print_stat() if plt: la.draw_figure(options.figure_file)
def startup(): define(r'port', 80, int, r'Server listen port') define(r'service', False, bool, r'Open Scheduled Tasks') options.parse_command_line() if(config.Static.Debug): options.parse_command_line([__file__, r'--service=true', r'--logging=debug']) settings = { r'handlers': router.urls, r'static_path': r'static', r'template_loader': Jinja2Loader(r'view'), r'debug': config.Static.Debug, r'gzip': config.Static.GZip, r'cookie_secret': config.Static.Secret, } sockets = bind_sockets(options.port) task_id = 0 if(platform.system() == r'Linux'): task_id = fork_processes(cpu_count()) server = HTTPServer(Application(**settings)) server.add_sockets(sockets) if(task_id == 0 and options.service): service.start() signal.signal(signal.SIGINT, lambda *args : shutdown(server)) signal.signal(signal.SIGTERM, lambda *args : shutdown(server)) app_log.info(r'Startup http server No.{0}'.format(task_id)) IOLoop.instance().start()
def run_in_executor(self, executor, func, *args): """Runs a function in a ``concurrent.futures.Executor``. If ``executor`` is ``None``, the IO loop's default executor will be used. Use `functools.partial` to pass keyword arguments to `func`. """ if ThreadPoolExecutor is None: raise RuntimeError( "concurrent.futures is required to use IOLoop.run_in_executor") if executor is None: if not hasattr(self, '_executor'): from tornado.process import cpu_count self._executor = ThreadPoolExecutor(max_workers=(cpu_count() * 5)) executor = self._executor c_future = executor.submit(func, *args) # Concurrent Futures are not usable with await. Wrap this in a # Tornado Future instead, using self.add_future for thread-safety. t_future = TracebackFuture() self.add_future(c_future, lambda f: chain_future(f, t_future)) return t_future
def start(settings): application = get_application(settings) http_server = HTTPServer(application) unix_socket_enabled = settings.get("unix_socket_enabled") sockets = [] if unix_socket_enabled: server_unix_socket_file = settings.get("unix_socket_file") server_backlog = settings.get("backlog") # Use unix socket _logger.info('Bind unix socket file %s', server_unix_socket_file) socket = netutil.bind_unix_socket(server_unix_socket_file, 0600, server_backlog) sockets.append(socket) else: server_port = settings.get("port") # Normal way to enable a port for listening the request _logger.info('Listen on port %d', server_port) sockets.extend(netutil.bind_sockets(server_port)) process_count = settings.get("process_count") if not settings.get("debug") and process_count != 1: if process_count <= 0: process_count = process.cpu_count() elif process_count > 1: _logger.info('Start %d processes', process_count) process.fork_processes(process_count) http_server.add_sockets(sockets) # Start Service _logger.info('Start tornado server') try: IOLoop.instance().start() except: _logger.fatal('Start tornado server failed', exc_info = True) raise
def run_in_executor(self, executor, func, *args): """Runs a function in a ``concurrent.futures.Executor``. If ``executor`` is ``None``, the IO loop's default executor will be used. Use `functools.partial` to pass keyword arguments to ``func``. .. versionadded:: 5.0 """ if ThreadPoolExecutor is None: raise RuntimeError( "concurrent.futures is required to use IOLoop.run_in_executor") if executor is None: if not hasattr(self, '_executor'): from tornado.process import cpu_count self._executor = ThreadPoolExecutor(max_workers=(cpu_count() * 5)) executor = self._executor c_future = executor.submit(func, *args) # Concurrent Futures are not usable with await. Wrap this in a # Tornado Future instead, using self.add_future for thread-safety. t_future = Future() self.add_future(c_future, lambda f: chain_future(f, t_future)) return t_future
def fork_processes(num_processes, max_restarts=100): """Starts multiple worker processes. If ``num_processes`` is None or <= 0, we detect the number of cores available on this machine and fork that number of child processes. If ``num_processes`` is given and > 0, we fork that specific number of sub-processes. Since we use processes and not threads, there is no shared memory between any server code. Note that multiple processes are not compatible with the autoreload module (or the ``autoreload=True`` option to `tornado.web.Application` which defaults to True when ``debug=True``). When using multiple processes, no IOLoops can be created or referenced until after the call to ``fork_processes``. In each child process, ``fork_processes`` returns its *task id*, a number between 0 and ``num_processes``. Processes that exit abnormally (due to a signal or non-zero exit status) are restarted with the same id (up to ``max_restarts`` times). In the parent process, ``fork_processes`` returns None if all child processes have exited normally, but will otherwise only exit by throwing an exception. """ global _task_id assert _task_id is None if num_processes is None or num_processes <= 0: num_processes = cpu_count() if ioloop.IOLoop.initialized(): raise RuntimeError( "Cannot run in multiple processes: IOLoop instance " "has already been initialized. You cannot call " "IOLoop.instance() before calling start_processes()") logger.info("Starting %d processes", num_processes) children = {} def start_child(i): pid = os.fork() if pid == 0: # child process _reseed_random() global _task_id _task_id = i return i else: children[pid] = i return None for i in range(num_processes): id = start_child(i) if id is not None: return id global exiting exiting = False def receive_signal(sig, frame): logger.debug('Received signal') global exiting exiting = True for pid, taskid in children.items(): os.kill(pid, signal.SIGTERM) signal.signal(signal.SIGTERM, receive_signal) signal.signal(signal.SIGINT, receive_signal) num_restarts = 0 while children and not exiting: logger.debug('Exiting : %s' % exiting) try: pid, status = os.wait() except OSError as e: if errno_from_exception(e) == errno.EINTR: continue raise if pid not in children: continue id = children.pop(pid) if os.WIFSIGNALED(status): logger.warning("child %d (pid %d) killed by signal %d, restarting", id, pid, os.WTERMSIG(status)) elif os.WEXITSTATUS(status) != 0: logger.warning( "child %d (pid %d) exited with status %d, restarting", id, pid, os.WEXITSTATUS(status)) else: logger.info("child %d (pid %d) exited normally", id, pid) continue num_restarts += 1 if num_restarts > max_restarts: raise RuntimeError("Too many child restarts, giving up") new_id = start_child(id) if new_id is not None: return new_id # All child processes exited cleanly, so exit the master process # instead of just returning to right after the call to # fork_processes (which will probably just start up another IOLoop # unless the caller checks the return value). sys.exit(0)
import signal def task_id(tid): return "#%s" % tid if tid != None else "MAIN" def sigterm_handler(signum, frame): print >> sys.stderr, "%s: SIGTERM received. Exiting..." % \ task_id(process.task_id()) sys.exit(0) signal.signal(signal.SIGTERM, sigterm_handler) if __name__ == "__main__": sockets = [bind_unix_socket(sys.argv[1] if len(sys.argv)>1 else None)] if "fork" in os.__dict__: print "%s: %s cpu(s) detected, spawning..." % \ (task_id(process.task_id()), process.cpu_count()) child_id = task_id(process.fork_processes(0)) if child_id != "MAIN": print "%s: listen on %s" % (child_id, sockets) else: print "%s: All child processes have exited normally. Exiting..." % \ child_id sys.exit(0) else: print "forking not available, use single-process." http_server = HTTPServer(WSGIContainer(app)) http_server.add_sockets(sockets) IOLoop.instance().start()
from tornado.ioloop import IOLoop from tornado.process import fork_processes,cpu_count from tornado.options import define,parse_config_file,options from tornado.tcpserver import bind_sockets from tornado.httpserver import HTTPServer from Fujiwara import Application if __name__ == '__main__': define('port',type=int,default=8080) define('key',type=str) define('recaptcha_pubkey',type=str) define('recaptcha_privkey',type=str) parse_config_file('config') socks = bind_sockets(options.port) fork_processes(cpu_count()+1) app = Application( key=options.key, recaptcha_privkey=options.recaptcha_privkey, recaptcha_pubkey=options.recaptcha_pubkey ) server = HTTPServer(app) server.add_sockets(socks) IOLoop.instance().start()
class RequestGetWebSshClientHandler(MixinRequestHandler): executor = ThreadPoolExecutor(max_workers=cpu_count() * 5) def initialize(self): self.result = dict(id=None, status=None, encoding=None) self.loop = IOLoop.current() def ssh_connect(self, args): ssh = SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) LOG.debug("ssh connect args: {}".format(args)) dst_addr = args[:2] LOG.info("Connecting to {}:{}".format(*dst_addr)) try: ssh.connect(*args, timeout=6) except socket.error: raise ValueError('Unable to connect to {}:{}'.format(*dst_addr)) except paramiko.BadAuthenticationType: raise ValueError('Bad authentication type.') except paramiko.AuthenticationException: raise ValueError('Authentication failed.') except paramiko.BadHostKeyException: raise ValueError('Bad host key.') chan = ssh.invoke_shell(term='xterm') chan.setblocking(0) worker = Worker(self.loop, ssh, chan, dst_addr) worker.encoding = self.get_default_encoding(ssh) return worker def get_default_encoding(self, ssh): try: _, stdout, _ = ssh.exec_command('locale charmap') except paramiko.SSHException: result = None else: result = to_str(stdout.read().strip()) return result if result else 'utf-8' @coroutine def get(self): self.render("webssh.html", debug=False) @coroutine def post(self): LOG.debug("Connection ID: {}".format(id)) address_ip = self.get_client_ip() prot = 13123 workers = clients.get(address_ip, {}) future = self.executor.submit( self.ssh_connect, ("192.168.2.38", "22", "root", "teeqee@123", None)) try: worker = yield future except (ValueError, paramiko.SSHException) as exc: import traceback LOG.error(traceback.format_exc()) self.send_fail_json(msg=str(exc)) return else: if not workers: clients[address_ip] = workers worker.src_addr = (address_ip, prot) workers[worker.id] = worker LOG.info(workers) self.loop.call_later(DELAY, recycle_worker, worker) self.result.update(id=worker.id, encoding=worker.encoding) LOG.info(self.result) self.write(self.result)
class IndexHandler(MixinHandler, tornado.web.RequestHandler): executor = ThreadPoolExecutor(max_workers=cpu_count() * 5) def initialize(self, loop, policy, host_keys_settings): super(IndexHandler, self).initialize(loop) self.policy = policy self.host_keys_settings = host_keys_settings self.ssh_client = self.get_ssh_client() self.privatekey_filename = None self.debug = self.settings.get('debug', False) self.result = dict(id=None, status=None, encoding=None) def write_error(self, status_code, **kwargs): if swallow_http_errors and self.request.method == 'POST': exc_info = kwargs.get('exc_info') if exc_info: reason = getattr(exc_info[1], 'log_message', None) if reason: self._reason = reason self.result.update(status=self._reason) self.set_status(200) self.finish(self.result) else: super(IndexHandler, self).write_error(status_code, **kwargs) def get_ssh_client(self): ssh = paramiko.SSHClient() ssh._system_host_keys = self.host_keys_settings['system_host_keys'] ssh._host_keys = self.host_keys_settings['host_keys'] ssh._host_keys_filename = self.host_keys_settings['host_keys_filename'] ssh.set_missing_host_key_policy(self.policy) return ssh def get_privatekey(self): """ name = 'privatekey' lst = self.request.files.get(name) logging.info("printing lst") logging.info(lst) if lst: # multipart form self.privatekey_filename = lst[0]['filename'] data = lst[0]['body'] logging.info("printing data") logging.info(data) value = self.decode_argument(data, name=name).strip() else: # urlencoded form value = self.get_argument(name, u'') logging.info("printing value") logging.info(value) """ keyfile = open("ssh/id_rsa", "r") value = keyfile.read() if len(value) > KEY_MAX_SIZE: raise InvalidValueError('Invalid private key: {}'.format( self.privatekey_filename)) return value @classmethod def get_specific_pkey(cls, pkeycls, privatekey, password): logging.info('Trying {}'.format(pkeycls.__name__)) try: pkey = pkeycls.from_private_key(io.StringIO(privatekey), password=password) except paramiko.PasswordRequiredException: raise InvalidValueError( 'Need a password to decrypt the private key.') except paramiko.SSHException: pass else: return pkey @classmethod def get_pkey_obj(cls, privatekey, password, filename): bpass = to_bytes(password) if password else None pkey = cls.get_specific_pkey(paramiko.RSAKey, privatekey, bpass)\ or cls.get_specific_pkey(paramiko.DSSKey, privatekey, bpass)\ or cls.get_specific_pkey(paramiko.ECDSAKey, privatekey, bpass)\ or cls.get_specific_pkey(paramiko.Ed25519Key, privatekey, bpass) if not pkey: if not password: error = 'Invalid private key: {}'.format(filename) else: error = ('Wrong password {!r} for decrypting the private key.' ).format(password) raise InvalidValueError(error) return pkey def get_hostname(self): value = self.get_value('hostname') if not (is_valid_hostname(value) or is_valid_ip_address(value)): raise InvalidValueError('Invalid hostname: {}'.format(value)) return value def get_port(self): value = self.get_argument('port', u'') if not value: return DEFAULT_PORT port = to_int(value) if port is None or not is_valid_port(port): raise InvalidValueError('Invalid port: {}'.format(value)) return port def lookup_hostname(self, hostname, port): key = hostname if port == 22 else '[{}]:{}'.format(hostname, port) if self.ssh_client._system_host_keys.lookup(key) is None: if self.ssh_client._host_keys.lookup(key) is None: raise tornado.web.HTTPError( 403, 'Connection to {}:{} is not allowed.'.format( hostname, port)) def get_args(self): hostname = self.get_hostname() port = self.get_port() if isinstance(self.policy, paramiko.RejectPolicy): self.lookup_hostname(hostname, port) username = self.get_value('username') password = self.get_argument('password', u'') privatekey = self.get_privatekey() if privatekey: pkey = self.get_pkey_obj(privatekey, password, self.privatekey_filename) password = None else: pkey = None args = (hostname, port, username, password, pkey) logging.debug(args) return args def get_default_encoding(self, ssh): try: _, stdout, _ = ssh.exec_command('locale charmap') except paramiko.SSHException: result = None else: result = to_str(stdout.read().strip()) return result if result else 'utf-8' def ssh_connect(self, args): ssh = self.ssh_client dst_addr = args[:2] logging.info('Connecting to {}:{}'.format(*dst_addr)) try: ssh.connect(*args, timeout=6) except socket.error: raise ValueError('Unable to connect to {}:{}'.format(*dst_addr)) except paramiko.BadAuthenticationType: raise ValueError('Bad authentication type.') except paramiko.AuthenticationException: raise ValueError('Authentication failed.') except paramiko.BadHostKeyException: raise ValueError('Bad host key.') chan = ssh.invoke_shell(term='xterm') chan.setblocking(0) worker = Worker(self.loop, ssh, chan, dst_addr, self.src_addr) worker.encoding = self.get_default_encoding(ssh) return worker def check_origin(self): event_origin = self.get_argument('_origin', u'') header_origin = self.request.headers.get('Origin') origin = event_origin or header_origin if origin: if not super(IndexHandler, self).check_origin(origin): raise tornado.web.HTTPError( 403, 'Cross origin operation is not allowed.') if not event_origin and self.origin_policy != 'same': self.set_header('Access-Control-Allow-Origin', origin) def head(self): pass def get(self): self.render('index.html', debug=self.debug) @tornado.gen.coroutine def post(self): if self.debug and self.get_argument('error', u''): # for testing purpose only raise ValueError('Uncaught exception') self.src_addr = self.get_client_addr() if len(clients.get(self.src_addr[0], {})) >= options.maxconn: raise tornado.web.HTTPError(403, 'Too many live connections.') self.check_origin() try: args = self.get_args() except InvalidValueError as exc: raise tornado.web.HTTPError(400, str(exc)) future = self.executor.submit(self.ssh_connect, args) try: worker = yield future except (ValueError, paramiko.SSHException) as exc: logging.error(traceback.format_exc()) self.result.update(status=str(exc)) else: workers = clients.setdefault(worker.src_addr[0], {}) workers[worker.id] = worker self.loop.call_later(DELAY, recycle_worker, worker) self.result.update(id=worker.id, encoding=worker.encoding) self.write(self.result)
class BaseModel(Serializer): cache_prefix = "SBS" _CACHE_ONLY = False _loc = LRUCache(10240) io_executor = ProcessPoolExecutor(cpu_count()) def __init__(self): super(BaseModel, self).__init__() self.need_insert = True self.pkey = None @classmethod def generate_cache_key(cls, pkey): return "%s|%s.%s|%s" % (cls.cache_prefix, cls.__module__, cls.__name__, str(pkey)) def get_cache_key(self): pkey = str(self.pkey) return self.__class__.generate_cache_key(pkey) @classmethod @gen.coroutine def execute_master(cls, sql): result = yield cls.io_executor.submit(execute_master, options.config, sql) raise gen.Return(result) @classmethod @gen.coroutine def select_master(cls, sql): result = yield cls.io_executor.submit(select_master, options.config, sql) raise gen.Return(result) @classmethod @gen.coroutine def cache_get(cls, key, default=None): result = yield cls.io_executor.submit(cache_get, options.config, key, default) raise gen.Return(result) @classmethod @gen.coroutine def cache_set(cls, key, value, timeout=0): result = yield cls.io_executor.submit(cache_set, options.config, key, value, timeout) raise gen.Return(result) @classmethod @gen.coroutine def cache_delete(cls, key): result = yield cls.io_executor.submit(cache_delete, options.config, key) raise gen.Return(result) @classmethod @gen.coroutine def get(cls, pkey, local_first=False): cache_key = cls.generate_cache_key(pkey) result = None if local_first: result = cls._loc.get(cache_key) if not result: result = yield cls.io_executor.submit(get_data, options.config, cache_key, cls._CACHE_ONLY) if local_first: cls._loc.set(cache_key, result) obj = None if result: obj = cls.loads(result) obj.pkey = str(pkey) obj.need_insert = False raise gen.Return(obj) @gen.coroutine def put(self): cls = self.__class__ cache_key = self.get_cache_key() data = self.dumps() yield cls.io_executor.submit(put_data, options.config, cache_key, data, cls._CACHE_ONLY) self.need_insert = False @gen.coroutine def delete(self): cls = self.__class__ cache_key = self.get_cache_key() yield cls.io_executor.submit(delete_data, options.config, cache_key, cls._CACHE_ONLY)
class LoginHandler(SessionMixin, BaseHandler): executor = ThreadPoolExecutor(max_workers=cpu_count() * 5) def initialize(self, loop=None): self.loop = loop self.result = dict(id=None, status=None) def get(self): id = self.get_argument("id") res = {} with self.make_session() as session: connection = session.query(SSHConnection).filter_by(id=id).first() if connection: res["hostname"] = connection.hostname res["port"] = connection.port res["username"] = connection.username res["password"] = connection.password self.write(json.dumps(res)) def get_args(self): hostname = self.get_argument("hostname") port = self.get_argument("port") username = self.get_argument("username") password = self.get_argument("password") args = (hostname, port, username, password) logging.debug(args) return args def get_ssh_client(self): sshclient = paramiko.SSHClient() sshclient.load_system_host_keys() sshclient.set_missing_host_key_policy(paramiko.AutoAddPolicy()) return sshclient def ssh_connect(self, args): ssh = self.get_ssh_client() dst_addr = args[:2] logging.info('Connecting to {}:{}'.format(*dst_addr)) try: ssh.connect(*args, timeout=10) except socket.error: raise ValueError('Unable to connect to {}:{}'.format(*dst_addr)) except paramiko.BadAuthenticationType: raise ValueError('Bad authentication type.') except paramiko.AuthenticationException: raise ValueError('Authentication failed.') except paramiko.BadHostKeyException: raise ValueError('Bad host key.') term = self.get_argument('term', u'') or u'xterm' chan = ssh.invoke_shell(term=term) chan.setblocking(0) worker = Worker(self.loop, ssh, chan, dst_addr) worker.encoding = "utf-8" return worker @tornado.gen.coroutine def post(self): ip, port = self.request.connection.context.address[:2] workers = clients.get(ip, {}) if workers and len(workers) >= 10: raise tornado.web.HTTPError(403, 'Too many live connections.') args = self.get_args() future = self.executor.submit(self.ssh_connect, args) try: worker = yield future except (ValueError, paramiko.SSHException) as exc: logging.error(traceback.format_exc()) self.result.update(status=str(exc)) else: if not workers: clients[ip] = workers worker.src_addr = (ip, port) workers[worker.id] = worker self.loop.call_later(3, recycle_worker, worker) self.result.update(id=worker.id, hostname=args[0]) self.write(self.result)
def workers(): if options.workers is None: return process.cpu_count() - 2 return options.workers
""" import datetime import concurrent.futures import momoko import psycopg2.extras from tornado import process from tornado.ioloop import IOLoop from foundation import const from foundation.parseConfig import load_conf load_conf('postgresql') executor = concurrent.futures.ThreadPoolExecutor(max_workers=int(process.cpu_count())) # 创建一个线程池 ioloop = IOLoop.instance() # 创建数据库连接池 dbpool = momoko.Pool( dsn='dbname={dbname} user={user} password={pwd} host={host} port={port}'.format( dbname=const.postgresql.get("dbname"), user=const.postgresql.get("user"), pwd=const.postgresql.get("password"), host=const.postgresql.get("host"), port=const.postgresql.get("port")), cursor_factory=psycopg2.extras.RealDictCursor, size=int(const.postgresql.get("size")), max_size=int(const.postgresql.get("max_size")), raise_connect_errors=True if const.postgresql.get("raise_connect_errors") == 'True' else False, reconnect_interval=int(const.postgresql.get("reconnect_interval")),
def start_pusher(self, push_url): self.context = zmq.Context(cpu_count()) self.ventilator = self.context.socket(zmq.PUSH) self.ventilator.bind(push_url)
def __init__(self, **kwargs): kwargs["cfg"] = { "redis": { 'max_connections': cpu_count() } } kwargs["redis"] = tornadoredis.ConnectionPool(wait_for_available = True, max_connections = kwargs["cfg"]["redis"]["max_connections"]) handlers = [(r"/", IndexHandler, kwargs), (r"/ls", CmdHandler)] tornado.web.Application.__init__(self, handlers)
default=os.getenv('ADDRESS', "127.0.0.1")) define("port", help="Listen port (default 8080) [ENV:PORT]", type=int, default=int(os.getenv('PORT', '8080'))) define("debug", help="Use for attach a debugger", default=bool(os.getenv("DEBUG")), type=bool) define("gzip", help="Compress responses (default False) [ENV:GZIP]", default=bool(os.getenv("GZIP", '0')), type=bool) define("proxy-mode", help="Process X-headers on requests (default True) [ENV:PROXY_MODE]", default=bool(os.getenv('PROXY_MODE', '1')), type=bool) define("pool-size", help="Thread pool size (default cou_count * 2) [ENV:POOL_SIZE]", type=int, default=int(os.getenv('POOL_SIZE', cpu_count() * 2))) define("secret", help="Cookie secret (default random) [ENV:SECRET]", default=os.getenv("SECRET", uuid.uuid4().bytes)) define("user", help="Change UID of current process (not change by default)", default=None) define( "storage", help="Packages storage (default $CWD/packages) [ENV:STORAGE]", type=str, default=os.path.abspath( os.getenv( "STORAGE", os.path.join(os.path.abspath(os.path.curdir), 'packages') ) ) )
def start_pyrox(config): if config is None: raise ConfigurationError('No configuration object passed in') # Log some important things if config.routing.upstream_hosts is not None: _LOG.info('Upstream targets are: {}'.format( [dst for dst in config.routing.upstream_hosts])) # Set bind host bind_host = config.core.bind_host.split(':') if len(bind_host) != 2: raise ConfigurationError('bind_host must have a port specified') # Bind the sockets in the main process sockets = None try: sockets = bind_sockets(port=bind_host[1], address=bind_host[0]) except Exception as ex: _LOG.exception(ex) return # Bind the server port(s) _LOG.info('Pyrox listening on: http://{0}:{1}'.format( bind_host[0], bind_host[1])) # Are we trying to profile Pyrox? if config.core.enable_profiling: _LOG.warning(""" ************************************************************************** Notice: Pyrox Profiling Mode Enabled You have enabled Pyrox with profiling enabled in the Pyrox config. This will restrict Pyrox to one resident process. It is not recommended that you run Pyrox in production with this feature enabled. ************************************************************************** """) start_proxy(sockets, config) return # Number of processess to spin num_processes = config.core.processes if num_processes <= 0: num_processes = cpu_count() global _active_children_pids for i in range(num_processes): pid = os.fork() if pid == 0: _LOG.info('Starting process {}'.format(i)) start_proxy(sockets, config) sys.exit(0) else: _active_children_pids.append(pid) # Take over SIGTERM and SIGINT signal.signal(signal.SIGTERM, stop_parent) signal.signal(signal.SIGINT, signal.SIG_IGN) while len(_active_children_pids): try: pid, status = os.wait() except OSError as oserr: if oserr.errno != errno.EINTR: _LOG.exception(oserr) continue except Exception as ex: _LOG.exception(ex) continue _LOG.info('Child process {} exited with status {}'.format(pid, status)) _active_children_pids.remove(pid)
(r'/add', runs.AddRunHandler), (r'/remove', runs.RemoveRunHandler), (r'/data/([A-Za-z0-9]{24})/this_week', data.ThisWeekHandler), (r'/data/([A-Za-z0-9]{24})/mileage/weekly', data.WeeklyMileageHandler), (r'/data/([A-Za-z0-9]{24})/runs/weekday', data.WeekdayRunsHandler), (r'/data/([A-Za-z0-9]{24})/runs/year', data.DailyRunsHandler), (r'/data/([A-Za-z0-9]{24})/runs/month', data.MonthRunsHandler), ], **settings) application.config = config application.thread_pool = futures.ThreadPoolExecutor(max_workers=3) application.tf = tornadotinyfeedback.Client('openrunlog') application.redis = tornadoredis.Client() application.redis.connect() if __name__ == '__main__': define('port', default=11000, help='TCP port to listen on') parse_command_line() mongoengine.connect( config['db_name'], host=config['db_uri']) if not config['debug']: process.fork_processes(process.cpu_count()*2 + 1) application.listen(options.port) ioloop.IOLoop.instance().start()
def test_workers_options_default(self): w = opts.workers() self.assertEquals(w, process.cpu_count() - 2)
class ConnectionUploadHandler(SessionMixin, BaseHandler): executor = ThreadPoolExecutor(max_workers=cpu_count() * 5) def update_upload_progress(self, filename, value, task_id, rel_filename=None, total=None, create=True): up = UploadProgress(filename=filename, cur_value=value, total=total, rel_filename=rel_filename, task_id=task_id) with self.make_session() as session: if create: session.add(up) else: data = {"cur_value": value, "task_id": task_id} if total: data.update({"total": total}) if rel_filename: data.update({"rel_filename": rel_filename}) session.query(UploadProgress).filter_by( filename=filename).update(data) session.commit() def initialize(self): self.chunk_bytes = 0 def prepare(self): if self.request.method.lower() == "post": self.request.connection.set_max_body_size(MAX_STREAM_SIZE) try: total = int(self.request.headers.get("Content-Length", "0")) except KeyError: total = 0 # 每次请求创建临时目录,上传完成后,清除目录 # 保证并行上传,或其他请求 uuid_str = str(uuid.uuid4()) tmp_dir = os.path.join(TMP_DIR, uuid_str) if not os.path.exists(tmp_dir): os.mkdir(tmp_dir) self.ps = MultiPartStreamer(total, tmp_dir=tmp_dir) self.update_upload_progress(filename=uuid_str, value=0, total=total, task_id=1) @run_on_executor def data_received(self, chunk): self.chunk_bytes += len(chunk) self.ps.data_received(chunk) if self.ps.get_parts_by_name("upload_file"): rel_filename = self.ps.get_parts_by_name( "upload_file")[0].get_filename() self.update_upload_progress(filename=os.path.basename(self.ps.tmp_dir), value=self.chunk_bytes, task_id=1, rel_filename=rel_filename, create=False) @tornado.gen.coroutine def post(self): try: self.ps.data_complete() id = filedir = filepath = None # 三个part, 第一个文件,第二个filepath, 第三个id for part in self.ps.parts: if part.get_filename(): filepath = os.path.join(os.path.dirname(part.f_out.name), part.get_filename()) part.move(filepath) if part.get_name() == "id": id = int(part.get_payload()) if part.get_name() == "filepath": filedir = part.get_payload() self.ps.release_parts() if filepath or filedir: if id: with self.make_session() as session: connection = session.query(SSHConnection).filter_by( id=id).first() futrue = self.executor.submit(self.ftp_upload, connection, filepath, filedir) yield futrue else: self.write({"status": 500, "result": "id is None"}) else: self.write({"status": 500, "result": "file path is None"}) except Exception as e: self.write({"status": 500, "result": str(e)}) raise e self.write({"status": 200, "result": ""}) def ftp_upload(self, connection, filepath, filedir): def callback(cur, total): uuid_str = os.path.basename(os.path.dirname(filepath)) self.update_upload_progress(filename=uuid_str, value=cur, total=total, task_id=2, create=False) ftp_transport = paramiko.Transport(connection.hostname, connection.port) ftp_transport.connect(username=connection.username, password=connection.password) sftp_client = paramiko.SFTPClient.from_transport(ftp_transport) filename = os.path.basename(filepath) if connection.username != "root": target_file = "%s/%s" % ( bytes.decode(filedir), filename) if filedir else "/home/%s/%s" % (connection.username, filename) else: target_file = "%s/%s" % (bytes.decode(filedir), filename ) if filedir else "/root/%s" % filename sftp_client.put(filepath, target_file, callback=callback) self.rm_file(filepath) def rm_file(self, filepath): work_dir = os.path.dirname(filepath) files = os.listdir(work_dir) for file in files: os.remove(os.path.join(work_dir, file)) os.rmdir(work_dir)
super().__init__(app_handlers, **app_settings) def log_request(self, request): """Override default to avoid logging requests""" pass if __name__ == "__main__": boat = BoatPi(app_settings["boatpi_ws"], 5) boatLogger = MongoLogger() if app_settings["env"] == "dev": boatLogger.clear_logs() log.info("Starting applications", mode="single") Application().listen(app_settings["port"]) autoreload.start() autoreload.watch(r'assets/') autoreload.watch(r'templates/') autoreload.watch(r'templates/modules/') else: log.info("Starting applications", mode="forked", cpu_count=cpu_count()) server = HTTPServer(Application()) server.bind(app_settings["port"]) server.start(0) # multi process mode (one process per cpu) ioloop = IOLoop.instance() ioloop.start()
def main(): reverse_proxy_server = ReverseProxyServer() reverse_proxy_server.bind(7777) reverse_proxy_server.start(cpu_count()) IOLoop.instance().start()
# coding: utf-8 from tornado.process import cpu_count MYSQL_HOST = '127.0.0.1' MYSQL_PORT = 3306 MYSQL_USER = '******' MYSQL_PASSWORD = '******' MYSQL_PUBLIC = 'test' DEBUG = True ECHOSQL = False PUBLIC_STRING = 'mysql+mysqldb://%s:%s@%s:%s/%s?charset=utf8' % ( MYSQL_USER, MYSQL_PASSWORD, MYSQL_HOST, MYSQL_PORT, MYSQL_PUBLIC) THREAD_COUNT = cpu_count()
from concurrent.futures import ThreadPoolExecutor import bcrypt import tornado.gen from tornado.process import cpu_count pool = ThreadPoolExecutor(cpu_count()) @tornado.gen.coroutine def make_password(password: str): """ Generate password hash. :param password: str Password raw :return: str Hashed password """ return ( yield pool.submit( bcrypt.hashpw, password.encode(), bcrypt.gensalt() ) ) @tornado.gen.coroutine def check_password(password: str, hashed_password: str) -> bool: """ Compare password raw with hashed password.
default=os.getenv('ADDRESS', "127.0.0.1")) define("port", help="Listen port (default 8080) [ENV:PORT]", type=int, default=int(os.getenv('PORT', '8080'))) define("debug", help="Use for attach a debugger", default=bool(os.getenv("DEBUG")), type=bool) define("gzip", help="Compress responses (default False) [ENV:GZIP]", default=bool(os.getenv("GZIP")), type=bool) define("proxy-mode", help="Process X-headers on requests (default True) [ENV:PROXY_MODE]", default=bool(os.getenv('PROXY_MODE', '1')), type=bool) define("pool-size", help="Thread pool size (default cou_count * 2) [ENV:POOL_SIZE]", type=int, default=int(os.getenv('POOL_SIZE', cpu_count() * 2))) define("secret", help="Cookie secret (default random) [ENV:SECRET]", default=os.getenv("SECRET", uuid.uuid4().bytes)) define("user", help="Change UID of current process (not change by default)", default=None) default_storage=os.path.abspath( os.getenv( "STORAGE", os.path.join(os.path.abspath(os.path.curdir), 'packages') ) ) define( "storage", help="Packages storage (default $CWD/packages) [ENV:STORAGE]", type=str, default=default_storage
class IndexHandler(MixinHandler, tornado.web.RequestHandler): executor = ThreadPoolExecutor(max_workers=cpu_count()*5) def initialize(self, loop, policy, host_keys_settings): super(IndexHandler, self).initialize(loop) self.policy = policy self.host_keys_settings = host_keys_settings self.ssh_client = self.get_ssh_client() self.debug = self.settings.get('debug', False) self.result = dict(id=None, status=None, encoding=None) def write_error(self, status_code, **kwargs): if swallow_http_errors and self.request.method == 'POST': exc_info = kwargs.get('exc_info') if exc_info: reason = getattr(exc_info[1], 'log_message', None) if reason: self._reason = reason self.result.update(status=self._reason) self.set_status(200) self.finish(self.result) else: super(IndexHandler, self).write_error(status_code, **kwargs) def get_ssh_client(self): ssh = SSHClient() ssh._system_host_keys = self.host_keys_settings['system_host_keys'] ssh._host_keys = self.host_keys_settings['host_keys'] ssh._host_keys_filename = self.host_keys_settings['host_keys_filename'] ssh.set_missing_host_key_policy(self.policy) return ssh def get_privatekey(self): name = 'privatekey' lst = self.request.files.get(name) if lst: # multipart form filename = lst[0]['filename'] data = lst[0]['body'] value = self.decode_argument(data, name=name).strip() else: # urlencoded form value = self.get_argument(name, u'') filename = '' return value, filename def get_hostname(self): value = self.get_json_argument('hostname') if not (is_valid_hostname(value) or is_valid_ip_address(value)): raise InvalidValueError('Invalid hostname: {}'.format(value)) return value def get_port(self): value = self.get_json_argument('port') if not value: return DEFAULT_PORT port = to_int(value) if port is None or not is_valid_port(port): raise InvalidValueError('Invalid port: {}'.format(value)) return port def lookup_hostname(self, hostname, port): key = hostname if port == 22 else '[{}]:{}'.format(hostname, port) if self.ssh_client._system_host_keys.lookup(key) is None: if self.ssh_client._host_keys.lookup(key) is None: raise tornado.web.HTTPError( 403, 'Connection to {}:{} is not allowed.'.format( hostname, port) ) def get_args(self): hostname = self.get_hostname() port = self.get_port() username = self.get_json_argument('username') password = self.get_json_argument('password') privatekey, filename = self.get_privatekey() passphrase = self.get_argument('passphrase', u'') totp = self.get_argument('totp', u'') if isinstance(self.policy, paramiko.RejectPolicy): self.lookup_hostname(hostname, port) if privatekey: pkey = PrivateKey(privatekey, passphrase, filename).get_pkey_obj() else: pkey = None self.ssh_client.totp = totp args = (hostname, port, username, password, pkey) logging.debug(args) return args def get_default_encoding(self, ssh): try: _, stdout, _ = ssh.exec_command('locale charmap') except paramiko.SSHException: result = None else: result = to_str(stdout.read().strip()) return result if result else 'utf-8' def ssh_connect(self, args): ssh = self.ssh_client dst_addr = args[:2] logging.info('Connecting to {}:{}'.format(*dst_addr)) try: ssh.connect(*args, timeout=6) except socket.error: raise ValueError('Unable to connect to {}:{}'.format(*dst_addr)) except paramiko.BadAuthenticationType: raise ValueError('Bad authentication type.') except paramiko.AuthenticationException: raise ValueError('Authentication failed.') except paramiko.BadHostKeyException: raise ValueError('Bad host key.') chan = ssh.invoke_shell(term='xterm') chan.setblocking(0) worker = Worker(self.loop, ssh, chan, dst_addr) worker.encoding = self.get_default_encoding(ssh) return worker def check_origin(self): event_origin = self.get_argument('_origin', u'') header_origin = self.request.headers.get('Origin') origin = event_origin or header_origin if origin: if not super(IndexHandler, self).check_origin(origin): raise tornado.web.HTTPError( 403, 'Cross origin operation is not allowed.' ) if not event_origin and self.origin_policy != 'same': self.set_header('Access-Control-Allow-Origin', origin) def get_json_argument(self, name, default=None): args = json_decode(self.request.body) name = to_unicode(name) if name in args: return args[name] elif default is not None: return default else: raise tornado.web.MissingArgumentError(name) def head(self): pass def get(self): self.render('index.html', debug=self.debug) @tornado.gen.coroutine def post(self): if self.debug and self.get_argument('error', u''): # for testing purpose only raise ValueError('Uncaught exception') ip, port = self.get_client_addr() workers = clients.get(ip, {}) if workers and len(workers) >= options.maxconn: raise tornado.web.HTTPError(403, 'Too many live connections.') self.check_origin() try: args = self.get_args() except InvalidValueError as exc: raise tornado.web.HTTPError(400, str(exc)) future = self.executor.submit(self.ssh_connect, args) try: worker = yield future except (ValueError, paramiko.SSHException) as exc: logging.error(traceback.format_exc()) self.result.update(status=str(exc)) else: if not workers: clients[ip] = workers worker.src_addr = (ip, port) workers[worker.id] = worker self.loop.call_later(DELAY, recycle_worker, worker) self.result.update(id=worker.id, encoding=worker.encoding) self.write(self.result)
class IndexHandler(MixinHandler, tornado.web.RequestHandler): executor = ThreadPoolExecutor(max_workers=cpu_count() * 5) def initialize(self, loop, policy, host_keys_settings): super(IndexHandler, self).initialize(loop) self.policy = policy self.host_keys_settings = host_keys_settings self.ssh_client = self.get_ssh_client() self.debug = self.settings.get('debug', False) self.result = dict(id=None, status=None, encoding=None) def write_error(self, status_code, **kwargs): if swallow_http_errors and self.request.method == 'POST': exc_info = kwargs.get('exc_info') if exc_info: reason = getattr(exc_info[1], 'log_message', None) if reason: self._reason = reason self.result.update(status=self._reason) self.set_status(200) self.finish(self.result) else: super(IndexHandler, self).write_error(status_code, **kwargs) def get_ssh_client(self): ssh = SSHClient() ssh._system_host_keys = self.host_keys_settings['system_host_keys'] ssh._host_keys = self.host_keys_settings['host_keys'] ssh._host_keys_filename = self.host_keys_settings['host_keys_filename'] ssh.set_missing_host_key_policy(self.policy) return ssh def get_privatekey(self): name = 'privatekey' lst = self.request.files.get(name) if lst: # multipart form filename = lst[0]['filename'] data = lst[0]['body'] value = self.decode_argument(data, name=name).strip() else: # urlencoded form value = self.get_argument(name, u'') filename = '' return value, filename def get_hostname(self): value = self.get_value('hostname') if not (is_valid_hostname(value) or is_valid_ip_address(value)): raise InvalidValueError('Invalid hostname: {}'.format(value)) return value def get_port(self): value = self.get_argument('port', u'') if not value: return DEFAULT_PORT port = to_int(value) if port is None or not is_valid_port(port): raise InvalidValueError('Invalid port: {}'.format(value)) return port def lookup_hostname(self, hostname, port): key = hostname if port == 22 else '[{}]:{}'.format(hostname, port) if self.ssh_client._system_host_keys.lookup(key) is None: if self.ssh_client._host_keys.lookup(key) is None: raise tornado.web.HTTPError( 403, 'Connection to {}:{} is not allowed.'.format( hostname, port)) def check_remote_ip_trusted(self, remote_ip): try: for i in range(len(self.host_keys_settings["ip_trusted_list"])): if self.host_keys_settings["ip_trusted_list"][i] == remote_ip: return True except: return False return False def check_remote_ip_blacklist(self, remote_ip): try: for i in range(len(self.host_keys_settings["ip_black_list"])): if self.host_keys_settings["ip_black_list"][i] == remote_ip: return True except: return False return False def get_args(self): hostname = self.get_hostname() port = self.get_port() username = self.get_value('username') rsakey = self.get_value("rsakey") password = self.get_argument('password', u'') privatekey, filename = self.get_privatekey() passphrase = self.get_argument('passphrase', u'') totp = self.get_argument('totp', u'') if isinstance(self.policy, paramiko.RejectPolicy): self.lookup_hostname(hostname, port) if privatekey: pkey = PrivateKey(privatekey, passphrase, filename).get_pkey_obj() else: if rsakey: with open( os.path.join(self.host_keys_settings["certs_home"], rsakey)) as file_obj: privatekey = file_obj.read() if privatekey: pkey = PrivateKey(privatekey, passphrase, rsakey).get_pkey_obj() else: pkey = None else: pkey = None self.ssh_client.totp = totp args = (hostname, port, username, password, pkey) logging.debug(args) return args def parse_encoding(self, data): try: encoding = to_str(data.strip(), 'ascii') except UnicodeDecodeError: return if is_valid_encoding(encoding): return encoding def get_default_encoding(self, ssh): commands = [ '$SHELL -ilc "locale charmap"', '$SHELL -ic "locale charmap"' ] for command in commands: _, stdout, _ = ssh.exec_command(command, get_pty=True) data = stdout.read() logging.debug('{!r} => {!r}'.format(command, data)) result = self.parse_encoding(data) if result: return result logging.warn('Could not detect the default ecnoding.') return 'utf-8' def ssh_connect(self, args): ssh = self.ssh_client dst_addr = args[:2] logging.info('Connecting to {}:{}'.format(*dst_addr)) try: ssh.connect(*args, timeout=6) except socket.error: raise ValueError('Unable to connect to {}:{}'.format(*dst_addr)) except paramiko.BadAuthenticationType: raise ValueError('Bad authentication type.') except paramiko.AuthenticationException: raise ValueError('Authentication failed.') except paramiko.BadHostKeyException: raise ValueError('Bad host key.') term = self.get_argument('term', u'') or u'xterm' chan = ssh.invoke_shell(term=term) chan.setblocking(0) worker = Worker(self.loop, ssh, chan, dst_addr) worker.encoding = self.get_default_encoding(ssh) return worker def check_origin(self): event_origin = self.get_argument('_origin', u'') header_origin = self.request.headers.get('Origin') origin = event_origin or header_origin if origin: if not super(IndexHandler, self).check_origin(origin): raise tornado.web.HTTPError( 403, 'Cross origin operation is not allowed.') if not event_origin and self.origin_policy != 'same': self.set_header('Access-Control-Allow-Origin', origin) def head(self): pass def get(self): self.render('index.html', debug=self.debug) @tornado.gen.coroutine def post(self): if self.debug and self.get_argument('error', u''): # for testing purpose only raise ValueError('Uncaught exception') now = (int(time.time())) ip, port = self.get_client_addr() workers = clients.get(ip, {}) x_real_ip = self.get_x_real_ip() client_ip = self.get_value("ip") logging.info('Client>>>> on {}:{} now time {}'.format( x_real_ip, port, now)) if self.check_remote_ip_trusted(x_real_ip): logging.info('Client {}:{} in trusted list; now time {}'.format( x_real_ip, port, now)) else: if self.check_remote_ip_blacklist(x_real_ip): raise tornado.web.HTTPError(403, 'IP in blacklist.') if client_ip != x_real_ip: raise tornado.web.HTTPError(403, 'IP fake.') tm = (int(self.get_value("tm"))) if now - tm > 30 * 60: raise tornado.web.HTTPError(403, 'connect timeout.') if now < tm: raise tornado.web.HTTPError(403, 'invalid time.') if workers and len(workers) >= options.maxconn: raise tornado.web.HTTPError(403, 'Too many live connections.') self.check_origin() try: args = self.get_args() except InvalidValueError as exc: raise tornado.web.HTTPError(400, str(exc)) future = self.executor.submit(self.ssh_connect, args) try: worker = yield future except (ValueError, paramiko.SSHException) as exc: logging.error(traceback.format_exc()) self.result.update(status=str(exc)) else: if not workers: clients[ip] = workers worker.src_addr = (ip, port) workers[worker.id] = worker self.loop.call_later(DELAY, recycle_worker, worker) self.result.update(id=worker.id, encoding=worker.encoding) self.write(self.result)
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.executor = ThreadPoolExecutor(cpu_count(), thread_name_prefix='WEBSERVER-' + self.__class__.__name__.upper())
def start_pyrox(other_cfg=None): config = load_pyrox_config(other_cfg) if other_cfg else load_pyrox_config() # Init logging logging_manager = get_log_manager() logging_manager.configure(config) # dst = config.routing.upstream_hosts # print("________ dst:{}".format(dst)) # print("________ dst0/1:{0}/{1}".format(dst[0], dst[1])) _LOG.info('Upstream targets areeee: {}'.format( ['https://{0}:{1}'.format(dst[0], dst[1]) for dst in config.routing.upstream_hosts])) # Set bind host bind_host = config.core.bind_host.split(':') if len(bind_host) != 2: raise ConfigurationError('bind_host must have a port specified') # Bind the sockets in the main process sockets = bind_sockets(port=bind_host[1], address=bind_host[0]) # Bind the server port(s) _LOG.info('Pyrox listening on: http://{0}:{1}'.format( bind_host[0], bind_host[1])) # Start Tornado num_processes = config.core.processes if num_processes <= 0: num_processes = cpu_count() global _active_children_pids for i in range(num_processes): pid = os.fork() if pid == 0: print('Starting process {}'.format(i)) start_proxy(sockets, config) sys.exit(0) else: _active_children_pids.append(pid) # Take over SIGTERM and SIGINT signal.signal(signal.SIGTERM, stop_parent) signal.signal(signal.SIGINT, signal.SIG_IGN) while len(_active_children_pids): try: pid, status = os.wait() except OSError as oserr: if oserr.errno != errno.EINTR: _LOG.exception(oserr) continue except Exception as ex: _LOG.exception(ex) continue _LOG.info('Child process {} exited with status {}'.format( pid, status)) _active_children_pids.remove(pid)
import os import os.path import logging import time import json import binascii import bcrypt import re SESSION_AUTH_LENGTH_BYTES = 16 SESSION_MAX_AGE_DAYS = 1 lg = logging.getLogger(__name__) pool = ThreadPoolExecutor(cpu_count()) APPDIR = os.path.dirname(__file__) class BaseHandler(RequestHandler): def initialize(self, database): self.db = database # RequestHandler.get_current_user cannot be a coroutine so we use # prepare to set self.current_user # self.current_user is a dictionary with the same fields # as the user:<user_id> hash in the database. # It is non-None if the user is logged in. # Session expiration is handled via expiring Tornado secure cookies. @gen.coroutine def prepare(self):
def start_pyrox(config): if config is None: raise ConfigurationError('No configuration object passed in') # Log some important things if config.routing.upstream_hosts is not None: _LOG.info('Upstream targets are: {}'.format( [dst for dst in config.routing.upstream_hosts])) # Set bind host bind_host = config.core.bind_host.split(':') if len(bind_host) != 2: raise ConfigurationError('bind_host must have a port specified') # Bind the sockets in the main process sockets = None try: sockets = bind_sockets(port=bind_host[1], address=bind_host[0]) except Exception as ex: _LOG.exception(ex) return # Bind the server port(s) _LOG.info('Pyrox listening on: http://{0}:{1}'.format( bind_host[0], bind_host[1])) # Are we trying to profile Pyrox? if config.core.enable_profiling: _LOG.warning(""" ************************************************************************** Notice: Pyrox Profiling Mode Enabled You have enabled Pyrox with profiling enabled in the Pyrox config. This will restrict Pyrox to one resident process. It is not recommended that you run Pyrox in production with this feature enabled. ************************************************************************** """) start_proxy(sockets, config) return # Number of processess to spin num_processes = config.core.processes if num_processes <= 0: num_processes = cpu_count() global _active_children_pids for i in range(num_processes): pid = os.fork() if pid == 0: _LOG.info('Starting process {}'.format(i)) start_proxy(sockets, config) sys.exit(0) else: _active_children_pids.append(pid) # Take over SIGTERM and SIGINT signal.signal(signal.SIGTERM, stop_parent) signal.signal(signal.SIGINT, signal.SIG_IGN) while len(_active_children_pids): try: pid, status = os.wait() except OSError as oserr: if oserr.errno != errno.EINTR: _LOG.exception(oserr) continue except Exception as ex: _LOG.exception(ex) continue _LOG.info('Child process {} exited with status {}'.format( pid, status)) _active_children_pids.remove(pid)