def update(self, obj, callback=None): """ Updates the value for a key atomically. Typical usage would be: c = etcd.Client() o = c.read("/somekey") o.value += 1 c.update(o) Args: obj (etcd.EtcdResult): The object that needs updating. """ assert isinstance(obj, EtcdResult), "obj not a EtcdResult." _log.debug("Updating %s to %s.", obj.key, obj.value) kwdargs = { 'dir': obj.dir, 'ttl': obj.ttl, 'prevExist': True } if not obj.dir: # prevIndex on a dir causes a 'not a file' error. d'oh! kwdargs['prevIndex'] = obj.modifiedIndex return self.write(obj.key, obj.value, callback=callback, **kwdargs)
def init_mysql_pool(self, jdbc_url, max_idle_conn=1, max_open_conn=10, max_recycle_sec=60): """ :param jdbc_url: mysql://root:xxx@xiaoqiang-zdm:3306/mysql :return: """ if not jdbc_url: return gen_log.debug("jdbc_url: %s", jdbc_url) conf = urlparse.urlparse(jdbc_url) gen_log.debug("hostname: %s, db: %s, user: %s, passwd: %s, port: %s", conf.hostname, conf.path, conf.username, conf.password, conf.port) db = '' if len(conf.path) > 1: db = conf.path[1:] return tornado_mysql.pools.Pool( dict( host=conf.hostname, port=conf.port, db=db, user=conf.username, passwd=conf.password, charset='utf8', cursorclass=tornado_mysql.cursors.DictCursor ), max_idle_connections=max_idle_conn, max_open_connections=max_open_conn, max_recycle_sec=max_recycle_sec )
def watch(self, key, index=None, timeout=None, recursive=None, callback=None): # todo """ Blocks until a new event has been received, starting at index 'index' Args: key (str): Key. index (int): Index to start from. timeout (int): max seconds to wait for a read. Returns: client.EtcdResult Raises: KeyValue: If the key doesn't exists. urllib3.exceptions.TimeoutError: If timeout is reached. >>> print client.watch('/key').value 'value' """ _log.debug("About to wait on key %s, index %s", key, index) if index: return self.read(key, wait=True, waitIndex=index, timeout=timeout, recursive=recursive, callback=callback) else: return self.read(key, wait=True, timeout=timeout, recursive=recursive, callback=callback)
def remove_handler(self, fd): self._handlers.pop(fd, None) self._events.pop(fd, None) try: self._impl.unregister(fd) except Exception: gen_log.debug("Error deleting fd from IOLoop", exc_info=True)
def _callback(fut): exc = fut.exc_info() if exc: if not isinstance(exc[1], etcdexcept.EtcdException): # We can't get the list of machines, if one server is in the # machines cache, try on it _log.error("Failed to get list of machines from %s%s: %r and retry it.", uri, self.version_prefix, exc) if self._machines_cache: self._base_url = self._machines_cache.pop(0) _log.debug("Retrying on %s", self._base_url) # Call myself self.ioloop.add_future(self.search_machine(), _callback) return else: raise etcdexcept.EtcdException("Could not get the list of servers, " "maybe you provided the wrong " "host(s) to connect to?") else: response = fut.result() machines = [ node.strip() for node in self._handle_server_response(response).body.decode('utf-8').split(',') ] _log.debug("Retrieved list of machines: %s", machines) self._machines_cache = machines if self._base_url not in self._machines_cache: self._base_url = self._choice_machine() callback(fut.result())
def close(self, all_fds=False): """Closes the IOLoop, freeing any resources used. If ``all_fds`` is true, all file descriptors registered on the IOLoop will be closed (not just the ones created by the IOLoop itself). Many applications will only use a single IOLoop that runs for the entire lifetime of the process. In that case closing the IOLoop is not necessary since everything will be cleaned up when the process exits. `IOLoop.close` is provided mainly for scenarios such as unit tests, which create and destroy a large number of IOLoops. An IOLoop must be completely stopped before it can be closed. This means that `IOLoop.stop()` must be called *and* `IOLoop.start()` must be allowed to return before attempting to call `IOLoop.close()`. Therefore the call to `close` will usually appear just after the call to `start` rather than near the call to `stop`. """ self.remove_handler(self._waker.fileno()) if all_fds: for fd in self._handlers.keys()[:]: try: os.close(fd) except Exception: gen_log.debug("error closing fd %s", fd, exc_info=True) self._waker.close() self._impl.close()
def fetch_impl(self, request, callback): self.queue.append((request, callback)) self._process_queue() if self.queue: gen_log.debug("max_clients limit reached, request queued. " "%d active, %d queued requests." % ( len(self.active), len(self.queue)))
def _on_content_headers(self, data, buf=b""): self._content_length_left -= len(data) data = self._boundary_buffer + data gen_log.debug("file header is %r", data) self._boundary_buffer = buf header_data = data[self._boundary_len + 2 :].decode("utf-8") headers = tornado.httputil.HTTPHeaders.parse(header_data) disp_header = headers.get("Content-Disposition", "") disposition, disp_params = tornado.httputil._parse_header(disp_header) if disposition != "form-data": gen_log.warning("Invalid multipart/form-data") self._read_content_body(None) if not disp_params.get("name"): gen_log.warning("multipart/form-data value missing name") self._read_content_body(None) name = disp_params["name"] if disp_params.get("filename"): ctype = headers.get("Content-Type", "application/unknown") fd, tmp_filename = tempfile.mkstemp(suffix=".tmp", prefix="tornado") self._request.files.setdefault(name, []).append( tornado.httputil.HTTPFile( filename=disp_params["filename"], tmp_filename=tmp_filename, content_type=ctype ) ) self._read_content_body(os.fdopen(fd, "wb")) else: gen_log.warning("multipart/form-data is not file upload, skipping...") self._read_content_body(None)
def process_message(self, client, msg): try: gen_log.debug("Received from WS: %s" % msg) msg = json.loads(msg) except ValueError: gen_log.warning("Received message not in json from client %s: %s" % (client.remote_address, msg)) return try: service_name = msg['service'] except KeyError: gen_log.warning("Malformed message from client %s: %s" % (client.remote_address, msg)) return try: service = self.services[service_name] except KeyError: gen_log.warning( 'Message for non existing service "%s" from client %s' % (service_name, client.remote_address)) return service.process_message(client, msg)
def init_mysql_pool(self, jdbc_url, max_connections=10, idle_seconds=60, wait_connection_timeout=3): """ :param jdbc_url: mysql://root:xxx@xiaoqiang-zdm:3306/mysql :return: """ if not jdbc_url: return gen_log.debug("jdbc_url: %s", jdbc_url) conf = urlparse.urlparse(jdbc_url) gen_log.debug("hostname: %s, db: %s, user: %s, passwd: %s, port: %s", conf.hostname, conf.path, conf.username, conf.password, conf.port) db = '' if len(conf.path) > 1: db = conf.path[1:] return tormysql.ConnectionPool( max_connections=int(max_connections), # max open connections idle_seconds=int(idle_seconds), # conntion idle timeout time, 0 is not timeout wait_connection_timeout=int(wait_connection_timeout), # wait connection timeout host=conf.hostname, user=conf.username, passwd=conf.password, db=db, charset="utf8" )
def resume_reading(self): """ Alias for adding the read interest to the event handler. """ if _SHOULD_LOG_DEBUG_OUTPUT: gen_log.debug('Resuming recv events for stream(fd:{0})'.format( self.fd)) self._add_event_interest(self._io_loop.READ)
def enable_errors(self): """ Alias for adding the error interest from the event handler. """ if _SHOULD_LOG_DEBUG_OUTPUT: gen_log.debug('Resuming error events for stream(fd:{})'.format( self.fileno)) self._add_event_interest(ERROR)
def enable_writes(self): """ Alias for adding the send interest to the event handler. """ if _SHOULD_LOG_DEBUG_OUTPUT: gen_log.debug('Resuming write events for stream(fd:{})'.format( self.fileno)) self._add_event_interest(WRITE)
def _handle_challenge(self, challenge): try: challenge_response = self.challenge_response(challenge) except ValueError: gen_log.debug("Malformed key data in WebSocket request") self._abort() return self._write_response(challenge_response)
def enable_reads(self): """ Alias for adding the read interest to the event handler. """ if _SHOULD_LOG_DEBUG_OUTPUT: gen_log.debug('Resuming read events for stream(fd:{0})'.format( self.fileno)) self._add_event_interest(READ)
def accept_connection(self): try: self._handle_websocket_headers() self._accept_connection() except ValueError: gen_log.debug("Malformed WebSocket request received", exc_info=True) self._abort() return
def disable_reading(self): """ Alias for removing the read interest from the event handler. """ if _SHOULD_LOG_DEBUG_OUTPUT: gen_log.debug('Halting read events for stream(fd:{})'.format( self.fd)) self._drop_event_interest(self._io_loop.READ)
def remove_handler(self, fd): """Stop listening for events on fd.""" self._handlers.pop(fd, None) self._events.pop(fd, None) try: self._impl.unregister(fd) except (OSError, IOError): gen_log.debug("Error deleting fd from IOLoop", exc_info=True)
def disable_errors(self): """ Alias for removing the error interest from the event handler. """ if _SHOULD_LOG_DEBUG_OUTPUT: gen_log.debug('Halting error events for stream(fd:{})'.format( self.fileno)) self._drop_event_interest(ERROR)
def disable_writes(self): """ Alias for removing the send interest from the event handler. """ if _SHOULD_LOG_DEBUG_OUTPUT: gen_log.debug('Halting write events for stream(fd:{})'.format( self.fileno)) self._drop_event_interest(WRITE)
def resume_writing(self): """ Alias for adding the send interest to the event handler. """ if _SHOULD_LOG_DEBUG_OUTPUT: gen_log.debug('Resuming send events for stream(fd:{})'.format( self.fd)) self._add_event_interest(self._io_loop.WRITE)
def on_finish(self): _id = self.get_query_argument('id') try: del self.application.storage[_id] except KeyError: pass gen_log.debug('#{_id}: cleanup.'.format(_id=_id))
def _execute(self, sql, args=None): pool = choice(SLAVE_TOR_MYSQL_POOL) gen_log.debug("TorMysqlClient _execute sql: %s, args: %s, pool: %s", sql, str(args), pool) with (yield pool.Connection()) as conn: with conn.cursor() as cursor: res = yield cursor.execute(sql, args) yield conn.commit() gen_log.debug("TorMysqlClient _execute res: %s", res) raise gen.Return(cursor)
def close(self, all_fds=False): self.remove_handler(self._waker.fileno()) if all_fds: for fd in self._handlers.keys()[:]: try: os.close(fd) except Exception: gen_log.debug("error closing fd %s", fd, exc_info=True) self._waker.close() self._impl.close()
def api_request(self, *path, **kwargs): """Make an API request""" url = url_path_join(self.url, 'api', *path) kwargs.setdefault('method', 'GET') kwargs['url'] = url gen_log.debug("%s %s", kwargs['method'], url) r = requests.request(**kwargs) r.raise_for_status() if r.text != '': return r.json()
def get_access_token(self): token = self.get_argument("access_token", "") if not token: try: token_str = self.request.headers.get("Authorization") token = token_str.replace("token ", "") except: token = "" gen_log.debug('token: "{}"'.format(token)) return token
def fetchall(self, sql, args=None): """ desc: 获取所有的数据 :param sql: :param args: :return: """ with (yield self._execute(sql, args)) as cursor: msg = cursor.fetchall() gen_log.debug("TorMysqlClient fetchall msg: %s", msg) raise gen.Return(msg)
def remove_handler(self, fd): """ 移除主循环中已存在的socket """ fd, obj = self.split_fd(fd) self._handlers.pop(fd, None) self._events.pop(fd, None) try: self._impl.unregister(fd) except Exception: gen_log.debug("Error deleting fd from IOLoop", exc_info=True)
def execute(self, sql, args=None): """ desc: 插入/更新/删除操作 :param sql: :param args: :return: """ pool = choice(MASTER_TORNADO_MYSQL_POOL) res = yield pool.execute(sql, args) gen_log.debug("TornadoMysqlClient insert res: %s", res) raise gen.Return(res)
def _close(self): """Close this stream.""" if self._socket is not None: gen_log.debug('Closing stream(fd: {})'.format(self.handle.fd)) self.handle.remove_handler() self._socket.close() self._socket = None if self._close_cb: self._run_callback(self._close_cb)
def close(self, all_fds=False): with self._callback_lock: self._closing = True self.remove_handler(self._waker.fileno()) if all_fds: for fd in self._handlers.keys(): try: close_method = getattr(fd, 'close', None) if close_method is not None: close_method() else: os.close(fd) except Exception: gen_log.debug("error closing fd %s", fd, exc_info=True) self._waker.close() self._impl.close()
def fetch_impl(self, request, callback): key = object() self.queue.append((key, request, callback)) if not len(self.active) < self.max_clients: timeout_handle = self.io_loop.add_timeout( self.io_loop.time() + min(request.connect_timeout, request.request_timeout), functools.partial(self._on_timeout, key, "in request queue")) else: timeout_handle = None self.waiting[key] = (request, callback, timeout_handle) self._process_queue() if self.queue: gen_log.debug("max_clients limit reached, request queued. " "%d active, %d queued requests." % (len(self.active), len(self.queue)))
def _on_headers(self, data): try: data = data.decode('latin1') eol = data.find("\r\n") start_line = data[:eol] try: method, uri, version = start_line.split(" ") except ValueError: raise tornado.httpserver._BadRequestException("Malformed HTTP request line") if not version.startswith("HTTP/"): raise tornado.httpserver._BadRequestException("Malformed HTTP version in HTTP Request-Line") headers = tornado.httputil.HTTPHeaders.parse(data[eol:]) # HTTPRequest wants an IP, not a full socket address if self.address_family in (socket.AF_INET, socket.AF_INET6): remote_ip = self.address[0] else: # Unix (or other) socket; fake the remote address remote_ip = '0.0.0.0' self._request = tornado.httpserver.HTTPRequest( connection=self, method=method, uri=uri, version=version, headers=headers, remote_ip=remote_ip, protocol=self.protocol) content_length = headers.get("Content-Length") if content_length: content_length = int(content_length) use_tmp_files = self._get_handler_info() if not use_tmp_files and content_length > self.stream.max_buffer_size: raise tornado.httpserver._BadRequestException("Content-Length too long") if headers.get("Expect") == "100-continue": self.stream.write(b"HTTP/1.1 100 (Continue)\r\n\r\n") if use_tmp_files: gen_log.debug('using temporary files for uploading') self._receive_content(content_length) else: gen_log.debug('using memory for uploading') self.stream.read_bytes(content_length, self._on_request_body) return self.request_callback(self._request) except tornado.httpserver._BadRequestException as e: gen_log.info("Malformed HTTP request from %s: %s", self.address[0], e) self.close() return
def _handle_server_response(self, response): """ Handles the server response """ if response.code in [200, 201]: return response else: _log.debug("Response %s", response.body) resp = response.body.decode('utf-8') # throw the appropriate exception try: r = json.loads(resp) r['status'] = response.code except (TypeError, ValueError): # Bad JSON, make a response locally. r = {"message": "Bad response", "cause": str(resp)} etcdexcept.EtcdError.handle(r)
def _callback(fut): exc = fut.exc_info() if exc: exc_obj = exc[1] if isinstance(exc_obj, HTTPError) and exc_obj.code < 500: result = self._handle_server_response(exc_obj.response) new_future.set_result(result) return elif (isinstance(exc_obj, HTTPError) and exc_obj.code > 500) or isinstance( exc_obj, socket.error): _log.error("Request to server %s failed: %r", self._base_url, exc_obj) if isinstance(params, dict) and params.get("wait") == "true": _log.debug("Watch timed out.") raise etcdexcept.EtcdWatchTimedOut( "Watch timed out: %r" % exc_obj, cause=exc_obj) if self._allow_reconnect and fut._retry: _log.info( "Reconnection allowed, looking for another " "server.") self._base_url = fut._retry.pop(0) fut_reconnect = func(self, path, method, params=params, timeout=timeout) fut_reconnect._retry = fut._retry self.ioloop.add_future(fut_reconnect, _callback) return else: _log.debug("Reconnection disabled, giving up.") raise etcdexcept.EtcdConnectionFailed( "Connection to etcd failed due to %r" % exc_obj, cause=exc_obj) else: _log.exception("Unexpected request failure.") result = fut.result() self._check_cluster_id(result) result = self._handle_server_response(fut.result()) new_future.set_result(result)
def get(self): """ Return file upload progress """ _id = self.get_query_argument('id') gen_log.debug('#{_id}: progress request.'.format(_id=_id)) try: progress = self.application.storage[_id] self.write({ 'id': _id, 'progress': progress, }) except KeyError: self.send_error(404)
async def init_container_info(self) -> bool: """ 必须在执行容器客户端之前显示调用 初始化容器信息 """ if self.container_id is None: self.container_is = False return True try: self.container_client = await self.docker_client.containers.get( self.container_id) return True except Exception as e: # 如果提供的id无法查询到容器则直接拒绝访问 gen_log.debug("The container could not be found") await self.send_message(code="400") return False
def accept_connection(self): try: self._handle_websocket_headers() except ValueError: self.handler.set_status(400) log_msg = "Missing/Invalid WebSocket headers" self.handler.finish(log_msg) gen_log.debug(log_msg) return try: self._accept_connection() except ValueError: gen_log.debug("Malformed WebSocket request received", exc_info=True) self._abort() return
def load_gettext_translations(directory: str, domain: str) -> None: """Loads translations from `gettext`'s locale tree Locale tree is similar to system's ``/usr/share/locale``, like:: {directory}/{lang}/LC_MESSAGES/{domain}.mo Three steps are required to have your app translated: 1. Generate POT translation file:: xgettext --language=Python --keyword=_:1,2 -d mydomain file1.py file2.html etc 2. Merge against existing POT file:: msgmerge old.po mydomain.po > new.po 3. Compile:: msgfmt mydomain.po -o {directory}/pt_BR/LC_MESSAGES/mydomain.mo """ import gettext global _translations global _supported_locales global _use_gettext _translations = {} for lang in os.listdir(directory): if lang.startswith("."): continue # skip .svn, etc if os.path.isfile(os.path.join(directory, lang)): continue try: os.stat( os.path.join(directory, lang, "LC_MESSAGES", domain + ".mo")) _translations[lang] = gettext.translation(domain, directory, languages=[lang]) except Exception as e: gen_log.error("Cannot load translation for '%s': %s", lang, str(e)) continue _supported_locales = frozenset( list(_translations.keys()) + [_default_locale]) _use_gettext = True gen_log.debug("Supported locales: %s", sorted(_supported_locales))
def load_gettext_translations(directory, domain): """从 `gettext` 的区域树加载翻译 区域树和系统的 ``/usr/share/locale`` 很类似, 例如:: {directory}/{lang}/LC_MESSAGES/{domain}.mo 让你的应用程序翻译有三步是必须的: 1. 生成POT翻译文件:: xgettext --language=Python --keyword=_:1,2 -d mydomain file1.py file2.html etc 2. 合并现有的POT文件:: msgmerge old.po mydomain.po > new.po 3. 编译:: msgfmt mydomain.po -o {directory}/pt_BR/LC_MESSAGES/mydomain.mo """ import gettext global _translations global _supported_locales global _use_gettext _translations = {} for lang in os.listdir(directory): if lang.startswith('.'): continue # skip .svn, etc if os.path.isfile(os.path.join(directory, lang)): continue try: os.stat( os.path.join(directory, lang, "LC_MESSAGES", domain + ".mo")) _translations[lang] = gettext.translation(domain, directory, languages=[lang]) except Exception as e: gen_log.error("Cannot load translation for '%s': %s", lang, str(e)) continue _supported_locales = frozenset( list(_translations.keys()) + [_default_locale]) _use_gettext = True gen_log.debug("Supported locales: %s", sorted(_supported_locales))
def send(self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None): """Sends Request object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple :param verify: (optional) Whether to verify SSL certificates. :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: trip.adapters.MessageDelegate """ future = Future() def callback(response): if isinstance(response, Exception): future.set_exception(response) else: future.set_result(response) key = object() request = (request, stream, timeout, verify, cert, proxies) self.queue.append((key, request, callback)) if not len(self.active) < self.max_clients: timeout_handle = self.io_loop.add_timeout( self.io_loop.time() + min(parse_timeout(timeout)), functools.partial(self._on_timeout, key, 'in request queue')) else: timeout_handle = None self.waiting[key] = (request, callback, timeout_handle) self._process_queue() if self.queue: gen_log.debug('max_clients limit reached, request queued. ' '%d active, %d queued requests.' % (len(self.active), len(self.queue))) return future
def fetch_impl(self, request, callback): key = object() self.queue.append((key, request, callback)) # 给队列里面添加一个相关的回调 if not len(self.active) < self.max_clients: # 如果活跃的太多 # 如果活跃的太多,保存一部分稍后处理 timeout_handle = self.io_loop.add_timeout( self.io_loop.time() + min(request.connect_timeout, request.request_timeout), functools.partial(self._on_timeout, key)) else: # 如果活跃的太少 timeout_handle = None # 添加一个到waiting self.waiting[key] = (request, callback, timeout_handle) self._process_queue() if self.queue: # 如果还有没有处理的,这里就是显示太多了 gen_log.debug("max_clients limit reached, request queued. " "%d active, %d queued requests." % (len(self.active), len(self.queue)))
def _insert(cls, entity: object) -> int: '''插入封装 :param cls: :param entity: :class:`BaseModel` ''' if not SESSION: gen_log.error('session is null') return ResultCode(0, '无法获取session对象,数据库可能连接错误') session = SESSION() try: session.add(entity) session.commit() session.close() gen_log.debug('add {entity} successed'.format(entity=entity)) return ResultCode(1, '插入成功') except Exception as e: gen_log.warning('add {entity} failed, error: {e}'.format( entity=entity, e=e)) session.close() return ResultCode(0, '插入失败')
def post(self): def _get_stix_package(ioc_xml): if ioc_xml is not None and len(ioc_xml) > 0: ns = namespaces.Namespace("http://openioc.org/openioc", "openioc", "") idgen.set_id_namespace(ns) stix_obj = to_stix(BytesIO(ioc_xml)) stix_package = STIXPackage_v1.from_xml( etree.fromstring(stix_obj.to_xml())) return stix_package else: raise RuntimeError('request body is empty.') try: msg = None for field_name, files in self.request.files.items(): for file in files: filename, content_type = file['filename'], file[ 'content_type'] body = file['body'] gen_log.debug('POST "%s" "%s" %d bytes', filename, content_type, len(body)) gen_log.debug('POST file body:\n"%s"', body) stix_package = _get_stix_package(body) if stix_package is not None: patterns = common.get_search_items(stix_package) msg = {'fields': patterns} self.http_normal(200, msg=msg if msg is not None else u'OK') except: trace_msg = traceback.format_exc().decode('utf-8') emsg = u'request_msg:{0} {1}'.format(self.request.body, trace_msg) gen_log.error(u',[session-id:{0}],{1}'.format(None, emsg)) self.http_error(400, msg=trace_msg)
def fetch_impl(self, request: HTTPRequest, callback: Callable[[HTTPResponse], None], future: asyncio.Future) -> None: key = object() self.queue.append((key, request, callback)) if not len(self.active) < self.max_clients: assert request.connect_timeout is not None assert request.request_timeout is not None timeout_handle = self.io_loop.add_timeout( self.io_loop.time() + min(request.connect_timeout, request.request_timeout), functools.partial(self._on_timeout, key, "in request queue"), ) else: timeout_handle = None self.waiting[key] = (request, callback, timeout_handle) self._process_queue() if self.queue: gen_log.debug("max_clients limit reached, request queued. " "%d active, %d queued requests." % (len(self.active), len(self.queue)))
def execute(cell, kernel, session): """Run a single cell, waiting for its output""" msg = session.msg('execute_request', content={ 'code': cell.source, 'user_expressions': [], 'silent': False, 'allow_stdin': False, }) msg['channel'] = 'shell' parent_id = msg['header']['msg_id'] ws = kernel['channels'] gen_log.debug("Executing:\n%s", cell.source) ws.write_message(json.dumps(msg, default=date_default)) output_done = False shell_done = False while not (output_done and shell_done): jmsg = yield ws.read_message() msg = json.loads(jmsg) if msg['channel'] == 'iopub': gen_log.debug("output:\n%s", json.dumps(msg['content'], indent=1)) if msg['msg_type'] == 'status' \ and msg['content']['execution_state'] == 'idle' \ and msg['parent_header']['msg_id'] == parent_id: output_done = True elif msg['channel'] == 'shell': gen_log.debug("reply:\n%s", json.dumps(msg['content'], indent=1)) shell_done = True else: gen_log.warn("Unrecognized channel: %s\n%s", msg['channel'], json.dumps(msg['content'], indent=1))
def password(cls, user_name, user_upwd, user_npwd, reset=0): """ #修改密码 :param user_name: 帐号 :param user_upwd: 原密码 :param user_npwd: 新密码 :param reset: 是否重置密码 """ if not SESSION: return 0, '数据库连接错误,请联系后台管理员', None err_msg = '' session = SESSION() try: if reset: # 重置密码 ret = session.query(cls).filter(user_name=user_name).update( { cls.user_upwd: user_npwd, cls.user_dpwd: user_npwd }, synchronize_session=False) else: # 修改密码 ret = session.query(cls).filter( and_( cls.user_name == user_name, or_(cls.user_upwd == user_upwd, cls.user_dpwd == user_upwd))).update( { cls.user_upwd: user_npwd, cls.user_dpwd: user_npwd }, synchronize_session=False) session.commit() session.close() gen_log.debug('ret: {0}'.format(ret)) return 200, '密码修改成功,请牢记,如果忘记请联系系统管理员', ret except Exception as e: err_msg = str(e) gen_log.error(str(e)) session.close() return 0, '密码修改失败: {0}'.format(err_msg), None
def accept_connection(self): try: self._handle_websocket_headers() except ValueError: gen_log.debug("Malformed WebSocket request received") self._abort() return scheme = self.handler.get_websocket_scheme() # draft76 only allows a single subprotocol subprotocol_header = '' subprotocol = self.request.headers.get("Sec-WebSocket-Protocol", None) if subprotocol: selected = self.handler.select_subprotocol([subprotocol]) if selected: assert selected == subprotocol subprotocol_header = "Sec-WebSocket-Protocol: %s\r\n" % selected # Write the initial headers before attempting to read the challenge. # This is necessary when using proxies (such as HAProxy), which # need to see the Upgrade headers before passing through the # non-HTTP traffic that follows. self.stream.write( tornado.escape.utf8( "HTTP/1.1 101 WebSocket Protocol Handshake\r\n" "Upgrade: WebSocket\r\n" "Connection: Upgrade\r\n" "Server: TornadoServer/%(version)s\r\n" "Sec-WebSocket-Origin: %(origin)s\r\n" "Sec-WebSocket-Location: %(scheme)s://%(host)s%(uri)s\r\n" "%(subprotocol)s" "\r\n" % (dict(version=tornado.version, origin=self.request.headers["Origin"], scheme=scheme, host=self.request.host, uri=self.request.uri, subprotocol=subprotocol_header)))) self.stream.read_bytes(8, self._handle_challenge)
def get(self, *args, **kwargs): self.open_args = args self.open_kwargs = kwargs # Upgrade header should be present and should be equal to WebSocket if self.request.headers.get("Upgrade", "").lower() != 'websocket': self.set_status(400) log_msg = "Can \"Upgrade\" only to \"WebSocket\"." self.finish(log_msg) gen_log.debug(log_msg) return # Connection header should be upgrade. # Some proxy servers/load balancers # might mess with it. # websockets握手 headers = self.request.headers connection = map(lambda s: s.strip().lower(), headers.get("Connection", "").split(",")) if 'upgrade' not in connection: self.set_status(400) log_msg = "\"Connection\" must be \"Upgrade\"." self.finish(log_msg) gen_log.debug(log_msg) return # Handle WebSocket Origin naming convention differences # The difference between version 8 and 13 is that in 8 the # client sends a "Sec-Websocket-Origin" header and in 13 it's # simply "Origin". if "Origin" in self.request.headers: origin = self.request.headers.get("Origin") else: origin = self.request.headers.get("Sec-Websocket-Origin", None) # If there was an origin header, check to make sure it matches # according to check_origin. When the origin is None, we assume it # did not come from a browser and that it can be passed on. if origin is not None and not self.check_origin(origin): self.set_status(403) log_msg = "Cross origin websockets not allowed" self.finish(log_msg) gen_log.debug(log_msg) return # 将此http连接detach,转为websocket使用 self.stream = self.request.connection.detach() self.stream.set_close_callback(self.on_connection_close) self.ws_connection = self.get_websocket_protocol() if self.ws_connection: self.ws_connection.accept_connection() else: if not self.stream.closed(): self.stream.write( tornado.escape.utf8( "HTTP/1.1 426 Upgrade Required\r\n" "Sec-WebSocket-Version: 7, 8, 13\r\n\r\n")) self.stream.close()
def main() -> None: parse_command_line() io_loop = IOLoop.instance() if options.debug: tornado.autoreload.start() for option, value in sorted(options.as_dict().items()): gen_log.debug(f"Option: {option}: {value}") ari_client = AriClient() io_loop.add_callback(ari_client.run) if options.ping_interval: ping_periodic_callback = PeriodicCallback(do_ping, options.ping_interval * 1000) ping_periodic_callback.start() io_loop.start()
def update(self, obj, callback=None): """ Updates the value for a key atomically. Typical usage would be: c = etcd.Client() o = c.read("/somekey") o.value += 1 c.update(o) Args: obj (etcd.EtcdResult): The object that needs updating. """ assert isinstance(obj, EtcdResult), "obj not a EtcdResult." _log.debug("Updating %s to %s.", obj.key, obj.value) kwdargs = {'dir': obj.dir, 'ttl': obj.ttl, 'prevExist': True} if not obj.dir: # prevIndex on a dir causes a 'not a file' error. d'oh! kwdargs['prevIndex'] = obj.modifiedIndex return self.write(obj.key, obj.value, callback=callback, **kwdargs)
def fetch_impl(self, request: HTTPRequest, callback: Callable[[HTTPResponse], None]) -> None: key = object() self.queue.append((key, request, callback)) assert request.connect_timeout is not None assert request.request_timeout is not None timeout_handle = None if len(self.active) >= self.max_clients: timeout = (min(request.connect_timeout, request.request_timeout) or request.connect_timeout or request.request_timeout) # min but skip zero if timeout: timeout_handle = self.io_loop.add_timeout( self.io_loop.time() + timeout, functools.partial(self._on_timeout, key, "in request queue"), ) self.waiting[key] = (request, callback, timeout_handle) self._process_queue() if self.queue: gen_log.debug("max_clients limit reached, request queued. " "%d active, %d queued requests." % (len(self.active), len(self.queue)))
def close(self, all_fds=False): """override to use *method* to close FDs instead of os.close on everything, which doesn't work on zmq Sockets. Should be fixed in a future tornado release. """ with self._callback_lock: self._closing = True self.remove_handler(self._waker.fileno()) if all_fds: for fd in self._handlers.keys(): try: # begin patch try: fd.close() except AttributeError: os.close(fd) # end patch except Exception: gen_log.debug("error closing fd %s", fd, exc_info=True) self._waker.close() self._impl.close()
def close(self, all_fds=False): with self._callback_lock: self._closing = True if all_fds: for fd in self._handlers: obj, _ = self._handlers[fd] if obj is not None and hasattr(obj, 'close'): try: obj.close() except Exception: gen_log.debug("error closing socket object %s", obj, exc_info=True) try: os.close(fd) except Exception: gen_log.debug("error closing fd %s", fd, exc_info=True) self._fdwaker.close() self._close_loop_handles() # Run the loop so the close callbacks are fired and memory is freed self._loop.run() self._loop = None
def post(self): """ Upload file """ _id = self.get_query_argument('id') uploaded_dir = self.application.settings['uploaded_dir'] buf_size = 4096 try: self.parser.finish_receive() for part in self.parser.parts: filename = part['headers'][0]['params']['filename'] tmpfile = part['tmpfile'] full_path = os.path.join(uploaded_dir, filename) with open(full_path, 'wb') as fp: tmpfile.seek(0) while True: buf = tmpfile.read(buf_size) if not buf: break fp.write(buf) self.set_header('Content-type', 'text/plain; charset=UTF-8') self.write( json.dumps({ 'id': _id, 'status': 'complete', 'download_url': u'/uploaded/{}'.format(filename) })) self.finish() finally: self.parser.release_parts() gen_log.debug('#{_id}: upload complete.'.format(_id=_id))
def _curl_debug(debug_type, debug_msg): debug_types = ('I', '<', '>', '<', '>') if debug_type == 0: gen_log.debug('%s', debug_msg.strip()) elif debug_type in (1, 2): for line in debug_msg.splitlines(): gen_log.debug('%s %s', debug_types[debug_type], line) elif debug_type == 4: gen_log.debug('%s %r', debug_types[debug_type], debug_msg)
def lock(cls, user_name, user_state): """ #加解锁 :param cls: :param user_name: 帐号 :param user_state: 1-解锁,0-加锁 """ if not SESSION: return '数据库连接错误,请联系后台管理员', None err_msg = '' session = SESSION() try: # 更新状态 ret = session.query(cls).filter(user_name=user_name).update( {cls.user_state: user_state}, synchronize_session=False) session.commit() session.close() gen_log.debug('ret: {0}'.format(ret)) return '加解锁成功', ret except Exception as e: err_msg = str(e) gen_log.error(str(e)) session.close() return '加解锁失败: {0}'.format(err_msg), None
def _curl_setup_request(curl, request, buffer, headers): curl.setopt(pycurl.URL, native_str(request.url)) # libcurl's magic "Expect: 100-continue" behavior causes delays # with servers that don't support it (which include, among others, # Google's OpenID endpoint). Additionally, this behavior has # a bug in conjunction with the curl_multi_socket_action API # (https://sourceforge.net/tracker/?func=detail&atid=100976&aid=3039744&group_id=976), # which increases the delays. It's more trouble than it's worth, # so just turn off the feature (yes, setting Expect: to an empty # value is the official way to disable this) if "Expect" not in request.headers: request.headers["Expect"] = "" # libcurl adds Pragma: no-cache by default; disable that too if "Pragma" not in request.headers: request.headers["Pragma"] = "" # Request headers may be either a regular dict or HTTPHeaders object if isinstance(request.headers, httputil.HTTPHeaders): curl.setopt(pycurl.HTTPHEADER, [native_str("%s: %s" % i) for i in request.headers.get_all()]) else: curl.setopt(pycurl.HTTPHEADER, [native_str("%s: %s" % i) for i in request.headers.items()]) if request.header_callback: curl.setopt(pycurl.HEADERFUNCTION, lambda line: request.header_callback(native_str(line))) else: curl.setopt(pycurl.HEADERFUNCTION, lambda line: _curl_header_callback(headers, native_str(line))) if request.streaming_callback: write_function = request.streaming_callback else: write_function = buffer.write if bytes_type is str: # py2 curl.setopt(pycurl.WRITEFUNCTION, write_function) else: # py3 # Upstream pycurl doesn't support py3, but ubuntu 12.10 includes # a fork/port. That version has a bug in which it passes unicode # strings instead of bytes to the WRITEFUNCTION. This means that # if you use a WRITEFUNCTION (which tornado always does), you cannot # download arbitrary binary data. This needs to be fixed in the # ported pycurl package, but in the meantime this lambda will # make it work for downloading (utf8) text. curl.setopt(pycurl.WRITEFUNCTION, lambda s: write_function(utf8(s))) curl.setopt(pycurl.FOLLOWLOCATION, request.follow_redirects) curl.setopt(pycurl.MAXREDIRS, request.max_redirects) curl.setopt(pycurl.CONNECTTIMEOUT_MS, int(1000 * request.connect_timeout)) curl.setopt(pycurl.TIMEOUT_MS, int(1000 * request.request_timeout)) if request.user_agent: curl.setopt(pycurl.USERAGENT, native_str(request.user_agent)) else: curl.setopt(pycurl.USERAGENT, "Mozilla/5.0 (compatible; pycurl)") if request.network_interface: curl.setopt(pycurl.INTERFACE, request.network_interface) if request.use_gzip: curl.setopt(pycurl.ENCODING, "gzip,deflate") else: curl.setopt(pycurl.ENCODING, "none") if request.proxy_host and request.proxy_port: curl.setopt(pycurl.PROXY, request.proxy_host) curl.setopt(pycurl.PROXYPORT, request.proxy_port) if request.proxy_username: credentials = '%s:%s' % (request.proxy_username, request.proxy_password) curl.setopt(pycurl.PROXYUSERPWD, credentials) else: curl.setopt(pycurl.PROXY, '') curl.unsetopt(pycurl.PROXYUSERPWD) if request.validate_cert: curl.setopt(pycurl.SSL_VERIFYPEER, 1) curl.setopt(pycurl.SSL_VERIFYHOST, 2) else: curl.setopt(pycurl.SSL_VERIFYPEER, 0) curl.setopt(pycurl.SSL_VERIFYHOST, 0) if request.ca_certs is not None: curl.setopt(pycurl.CAINFO, request.ca_certs) else: # There is no way to restore pycurl.CAINFO to its default value # (Using unsetopt makes it reject all certificates). # I don't see any way to read the default value from python so it # can be restored later. We'll have to just leave CAINFO untouched # if no ca_certs file was specified, and require that if any # request uses a custom ca_certs file, they all must. pass if request.allow_ipv6 is False: # Curl behaves reasonably when DNS resolution gives an ipv6 address # that we can't reach, so allow ipv6 unless the user asks to disable. # (but see version check in _process_queue above) curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4) else: curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_WHATEVER) # Set the request method through curl's irritating interface which makes # up names for almost every single method curl_options = { "GET": pycurl.HTTPGET, "POST": pycurl.POST, "PUT": pycurl.UPLOAD, "HEAD": pycurl.NOBODY, } custom_methods = set(["DELETE", "OPTIONS", "PATCH"]) for o in curl_options.values(): curl.setopt(o, False) if request.method in curl_options: curl.unsetopt(pycurl.CUSTOMREQUEST) curl.setopt(curl_options[request.method], True) elif request.allow_nonstandard_methods or request.method in custom_methods: curl.setopt(pycurl.CUSTOMREQUEST, request.method) else: raise KeyError('unknown method ' + request.method) # Handle curl's cryptic options for every individual HTTP method if request.method in ("POST", "PUT"): if request.body is None: raise AssertionError( 'Body must not be empty for "%s" request' % request.method) request_buffer = BytesIO(utf8(request.body)) curl.setopt(pycurl.READFUNCTION, request_buffer.read) if request.method == "POST": def ioctl(cmd): if cmd == curl.IOCMD_RESTARTREAD: request_buffer.seek(0) curl.setopt(pycurl.IOCTLFUNCTION, ioctl) curl.setopt(pycurl.POSTFIELDSIZE, len(request.body)) else: curl.setopt(pycurl.INFILESIZE, len(request.body)) elif request.method == "GET": if request.body is not None: raise AssertionError('Body must be empty for GET request') if request.auth_username is not None: userpwd = "%s:%s" % (request.auth_username, request.auth_password or '') if request.auth_mode is None or request.auth_mode == "basic": curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC) elif request.auth_mode == "digest": curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_DIGEST) else: raise ValueError("Unsupported auth_mode %s" % request.auth_mode) curl.setopt(pycurl.USERPWD, native_str(userpwd)) gen_log.debug("%s %s (username: %r)", request.method, request.url, request.auth_username) else: curl.unsetopt(pycurl.USERPWD) gen_log.debug("%s %s", request.method, request.url) if request.client_cert is not None: curl.setopt(pycurl.SSLCERT, request.client_cert) if request.client_key is not None: curl.setopt(pycurl.SSLKEY, request.client_key) if threading.activeCount() > 1: # libcurl/pycurl is not thread-safe by default. When multiple threads # are used, signals should be disabled. This has the side effect # of disabling DNS timeouts in some environments (when libcurl is # not linked against ares), so we don't do it when there is only one # thread. Applications that use many short-lived threads may need # to set NOSIGNAL manually in a prepare_curl_callback since # there may not be any other threads running at the time we call # threading.activeCount. curl.setopt(pycurl.NOSIGNAL, 1) if request.prepare_curl_callback is not None: request.prepare_curl_callback(curl)
def post(self): def _get_stix_package(in_json): if in_json is not None and len(in_json) > 0: initialize_options() set_option_value("version_of_stix2x", stix_version) container = stixmarx.new() stix_package = container.package json_content = json.loads(in_json.decode()) #.decode('utf-8')) if type(json_content) == list: for json_data in json_content: if "type" in json_data and json_data[ "type"] == "indicator": indicator = convert_indicator(json_data) stix_package.add_indicator(indicator) else: if "type" in json_content and json_content[ "type"] == "bundle": if "objects" in json_content and json_content[ "objects"] and type( json_content["objects"]) == list: for json_data in json_content["objects"]: #print("[json_data]") #print(json_data) if "type" in json_data and json_data[ "type"] == "indicator": indicator = convert_indicator(json_data) stix_package.add_indicator(indicator) elif "type" in json_content and json_content[ "type"] == "indicator": indicator = convert_indicator(json_content) stix_package.add_indicator(indicator) container.flush() container = None return stix_package else: raise RuntimeError('request body is empty.') try: msg = None for field_name, files in self.request.files.items(): for file in files: filename, content_type = file['filename'], file[ 'content_type'] body = file['body'] #print(body) gen_log.debug('POST "%s" "%s" %d bytes', filename, content_type, len(body)) gen_log.debug('POST file body:\n"%s"', body) stix_package = _get_stix_package(body) #print(stix_package) if stix_package is not None: patterns = common.get_search_items(stix_package) msg = {'fields': patterns} #print(msg) self.http_normal(200, msg=msg if msg is not None else u'OK') except: trace_msg = traceback.format_exc() #.decode('utf-8') emsg = u'request_msg:{0} {1}'.format(self.request.body, trace_msg) gen_log.error(u',[session-id:{0}],{1}'.format(None, emsg)) self.http_error(400, msg=trace_msg)