def append_binary_to_file(file_name, bin_buf): """ Write to the specified filename, the provided binary buffer. Create the file if required. :param file_name: File name. :type file_name: str :param bin_buf: Binary buffer to write. :type bin_buf: bytes :return: The number of bytes written or lt 0 if error. :rtype int """ # Go rd = None try: # Open (text : open return a io.BufferedReader) rd = open(file_name, "ab+") # Read everything return rd.write(bin_buf) except IOError as e: # Exception... logger.error("append_binary_to_file : IOError, ex=%s", SolBase.extostr(e)) return -1 except Exception as e: logger.error("append_binary_to_file : Exception, ex=%s", SolBase.extostr(e)) return -1 finally: # Close if not None... if rd: rd.close()
def safe_close_socket(cls, soc_to_close): """ Safe close a socket :param cls: cls :param soc_to_close: socket :return: Nothing """ if soc_to_close is None: return try: soc_to_close.shutdown(2) except Exception as e: logger.debug("Socket shutdown ex=%s", SolBase.extostr(e)) try: soc_to_close.close() except Exception as e: logger.debug("Socket close ex=%s", SolBase.extostr(e)) try: del soc_to_close except Exception as e: logger.debug("Socket del ex=%s", SolBase.extostr(e))
def file_to_binary(file_name): """ Load a file toward a binary buffer. :param file_name: File name. :type file_name: str :return: Return the binary buffer or None in case of error. :rtype: bytes,None """ # Check if not FileUtility.is_file_exist(file_name): logger.error("file_to_binary : file_name not exist, file_name=%s", file_name) return None # Go rd = None try: # Open (binary : open return a io.BufferedReader) rd = open(file_name, "rb") # Read everything return rd.read() except IOError as e: # Exception... logger.error("IOError, ex=%s", SolBase.extostr(e)) return None except Exception as e: logger.error("Exception, ex=%s", SolBase.extostr(e)) return None finally: # Close if not None... if rd: rd.close()
def connect(self): """ Connect to server. :return Return true upon success. """ # Stats Meters.aii("ping.client.client_connect_count") # Call base dt_start = SolBase.datecurrent() b = TcpSimpleClient.connect(self) if not b: # Stat logger.error("PingSimpleClient : connect failed, fatal, exiting") Meters.aii("ping.client.client_connect_error") # Exit return False # Stat Meters.dtci("ping.client.delay_client_connect", SolBase.datediff(dt_start)) # SSL stats ms = self._get_ssl_handshake_ms() if ms: Meters.dtci("ping.client.delay_client_sslhandshake", ms) # Send hello self._protocol_client_hello_send() return True
def test_meters_bench(self): """ Test """ # For logs format SolBase.sleep(100) logger.info("Bench now") Meters.reset() self._bench(1, "aii_tags_NO", Meters.aii, "aii1", 1, None) Meters.reset() self._bench(1, "aii_tags_go", Meters.aii, "aii1", 1, {"T1": "V1", "T2": "V2"}) Meters.reset() self._bench(1, "dtc_tags_NO", Meters.dtci, "aii1", 0.1, 1, None) Meters.reset() self._bench(1, "dtc_tags_go", Meters.dtci, "aii1", 0.1, 1, {"T1": "V1", "T2": "V2"}) Meters.reset() self._bench(1, "dtc_tags_big_NO", Meters.dtci, "aii1", 1000000000, 1, None) Meters.reset() self._bench(1, "dtc_tags_big_go", Meters.dtci, "aii1", 1000000000, 1, {"T1": "V1", "T2": "V2"}) # For logs format SolBase.sleep(100) logger.info("Bench over")
def start(self): """ Start """ with self._locker: try: lifecyclelogger.info("Start : starting") # Check if self._is_running: logger.warning("Already running, doing nothing") # Start self._server_greenlet = gevent.spawn(self._server_forever) SolBase.sleep(0) # Wait lifecyclelogger.debug("Start : waiting") self._start_event.wait() SolBase.sleep(0) # Signal self._is_running = True lifecyclelogger.info("Start : started") except Exception as e: logger.error("Exception, e=%s", SolBase.extostr(e))
def _logging_reset(self): """ Logging reset """ # Go # Ouch, this hack disable console logs (zzzz), status invocation now flush nothing... if self.vars and "action" in self.vars and self.vars["action"] in [ "status", "reload", "stop" ]: logger.debug( "Bypassing switch to logfile due to 'status|reload|stop' action" ) else: logger.debug( "Switching to logfile, you will lost console logs now") for h in logging.root.handlers: h.close() SolBase.logging_init( log_level=self._loglevel, force_reset=True, log_to_file=self.v_log_to_file, log_to_syslog=self.v_log_to_syslog, log_to_syslog_facility=self.v_log_to_syslog_facility, log_to_console=self.v_log_to_console, )
def _server_forever(self): """ Exec loop """ try: # Alloc logger.info("Allocating WSGIServer") self._wsgi_server = WSGIServer(listener=('localhost', 7900), application=self.on_request) logger.info("Starting, %s, %s", self._wsgi_server.address, _parse_address(self._wsgi_server.address)) SolBase.sleep(0) # Signal logger.info("Signaling _start_event") self._start_event.set() SolBase.sleep(0) # This will block until signaled logger.info("Calling serve_forever") self._wsgi_server.serve_forever() except Exception as e: logger.error("Ex=%s", SolBase.extostr(e)) # This is fatal, we exit, we cannot serve exit(-1) finally: logger.info("Clearing _start_event") self._start_event.clear()
def generate_server_keys(cls): """ Generate server keys :return: Path to server keys. """ try: # Get current dir current_dir = dirname( abspath(__file__)) + SolBase.get_pathseparator() # Certificates path cert_path = current_dir + "Certificates" + SolBase.get_pathseparator( ) # Copy to /tmp (required for some files) shutil.copyfile(cert_path + "server.key", "/tmp/server.key") shutil.copyfile(cert_path + "server.crt", "/tmp/server.crt") # Ok return cert_path except Exception as e: logger.error("generate_server_keys : Exception, ex=%s", SolBase.extostr(e)) return None
def _remove_client_asynch(self, client_id): """ Remove a client, asynch. :param client_id: The client id. :type client_id: int """ # Spawn logger.debug("entering, client_id=%s", client_id) # Signal event (mantis 1280) evt = Event() # Spawn gevent.spawn(self._remove_client, client_id, evt) # Switch SolBase.sleep(0) # And wait # Note : remove this wait do not impact unittest... logger.debug("waiting, client_id=%s", client_id) evt.wait() # Over logger.debug("done, client_id=%s", client_id)
def send_binary_to_socket_with_signal(self, signaled_buffer): """ Send to socket, asynch. Upon completion, signaled_buffer.send_event is set. Caution : Caller MUST check the boolean returned. If False, the event will NOT be set. :param signaled_buffer: A signaled_buffer instance. :type signaled_buffer: pysoltcp.tcpbase.SignaledBuffer.SignaledBuffer :return: True is send has been scheduled, false otherwise. :rtype bool """ # Check if not isinstance(signaled_buffer, SignaledBuffer): logger.error("signaled_buffer not a SignaledBuffer, class=%s, self=%s", SolBase.get_classname(signaled_buffer), self) return False # Check if not isinstance(signaled_buffer.binary_buffer, binary_type): logger.error("binary_buffer not a binary, class=%s, self=%s", SolBase.get_classname(signaled_buffer.binary_buffer), self) return False # Check if not self.__is_running(): logger.debug( "not connected, returning false, self=%s", self) return False # Enqueue self.__send_queue.put(signaled_buffer) # Stats Meters.aii("tcp.server.server_bytes_send_pending", len(signaled_buffer.binary_buffer)) return True
def _write_to_socket(self, local_buffer): """ Write to the socket. """ # Check if not self.__is_running(): logger.debug("not connected, doing nothing, self=%s", self) return False try: # self.current_socket.sendall(local_buffer) return True except error as e: # [Errno 11] Resource temporarily unavailable # Means that nothing is available to read. # If not this, we raise. if e.args[0] != EWOULDBLOCK: # Raise :) logger.debug("Exception, ex=%s, self=%s", SolBase.extostr(e), self) self._disconnect_helper("_write_to_socket : error / No EWOULDBLOCK") return False else: # Normal logger.debug("normal exception/EWOULDBLOCK, ex=%s, self=%s", e, self) return False except Exception as e: logger.info("Exception, ex=%s, self=%s", SolBase.extostr(e), self) self._disconnect_helper("_write_to_socket : Exception") return False
def _schedule_next_watchdog(self): """ Schedule next run. """ try: # Started ? if not self._is_started: return with self._run_lock: # Re-check if not self._is_started: return # Yes, schedule self._watchdog_greenlet = gevent.spawn_later( self._watchdog_interval_ms * 0.001, self._watchdog_run) # Wait SolBase.sleep(0) except Exception as e: logger.error("_schedule_next_watchdog : Exception, e=%s", SolBase.extostr(e)) finally: pass
def handle(self, data, address): """ Handle one udp message :param data: data :type data: str :param address: address :type address: str """ ms_start = SolBase.mscurrent() try: # Handle data pass # Stats Meters.aii("resolvusclient_udp_recv") except Exception as e: # Log logger.warning( "Handle failed, data_len=%s, address=%s, data=%s, ex=%s", len(data), repr(address), repr(data), SolBase.extostr(e)) # Stat Meters.aii("resolvusclient_udp_recv_ex") finally: Meters.dtci("resolvusclient_udp_recv_dtc", SolBase.msdiff(ms_start))
def test_basic_eviction_with_get_ttl(self): """ Test :return: """ # Alloc self.mem_cache = MemoryCache(cb_evict=self.eviction_callback) # Put self.mem_cache.put("keyA", b"valA", 60000) self.mem_cache.put("keyB", b"valB", 500) logger.info("ms cur=%s", SolBase.mscurrent()) logger.info("A : %s", self.mem_cache.get_raw("keyA")) logger.info("B : %s", self.mem_cache.get_raw("keyB")) # Wait a bit SolBase.sleep(600) logger.info("ms after sleep=%s", SolBase.mscurrent()) # A : must be present # B : must be evicted (TTL elapsed) self.assertEqual(self.mem_cache.get("keyA"), b"valA") self.assertIsNone(self.mem_cache.get("keyB")) self.assertEqual(self.evict_count, 1) self.assertEqual(self.evict_last_key, "keyB") self.assertEqual(self.evict_last_value, b"valB") self.assertEqual(Meters.aig("mcs.cache_evict_ttl_get"), 1) # Stop self.mem_cache.stop_cache() self.mem_cache = None
def _bench(self, per_sec_multi, key, method_to_call, *args): """ Internal bench :param per_sec_multi: Multiply per_sec (case of lists) :type per_sec_multi: int :param key: For log :type key: str :param method_to_call: Method point :type callable :param args: args :type args: object """ if isinstance(args[0], list): # noinspection PyArgumentList,PyTypeChecker logger.debug("Using list, len=%s", len(args[0])) i = 0 loop = 0 per_loop = self.per_loop ms = SolBase.mscurrent() while SolBase.msdiff(ms) < self.max_ms: for _ in range(0, per_loop): method_to_call(*args) i += self.per_loop loop += 1 i = i * per_sec_multi ms = SolBase.msdiff(ms) sec = ms / 1000.0 per_sec = i / sec ms = round(ms, 2) sec = round(sec, 2) per_sec = round(per_sec, 2) logger.info("%32s, loop=%8s, i=%8s, ms=%8.2f, sec=%6.2f, per_sec=%12.2f", key, loop, i, ms, sec, per_sec)
def test_basic_ttl(self): """ Test :return: """ # Alloc self.redis_cache = RedisCache() # Put self.redis_cache.put(self.key_prefix + "keyA", b"valA", 60000) self.redis_cache.put(self.key_prefix + "keyB", b"valB", 1000) logger.info("ms cur=%s", SolBase.mscurrent()) logger.info("A : %s", self.redis_cache.get(self.key_prefix + "keyA")) logger.info("B : %s", self.redis_cache.get(self.key_prefix + "keyB")) # Wait a bit SolBase.sleep(2000) logger.info("ms after sleep=%s", SolBase.mscurrent()) # A : must be present # B : must be evicted (TTL elapsed) self.assertEqual(self.redis_cache.get(self.key_prefix + "keyA"), b"valA") self.assertIsNone(self.redis_cache.get(self.key_prefix + "keyB")) self.assertEqual(Meters.aig("rcs.cache_put"), 2) self.assertEqual(Meters.aig("rcs.cache_get_hit"), 3) self.assertEqual(Meters.aig("rcs.cache_get_miss"), 1) # Stop self.redis_cache.stop_cache() self.redis_cache = None
def get(self, key): """ Get from cache. :param key: Any key :type key: str :return An obj or null if not in cache :rtype bytes, None """ ms_start = SolBase.mscurrent() try: if not isinstance(key, (bytes, str)): raise Exception("Key must be (bytes, str)") # Use read redis v = self._read_redis.get(key) if v: Meters.aii(self.meters_prefix + "rcs.cache_get_hit") else: Meters.aii(self.meters_prefix + "rcs.cache_get_miss") return v except Exception as e: logger.warning("Exception, ex=%s", SolBase.extostr(e)) Meters.aii(self.meters_prefix + "rcs.cache_ex") return None finally: Meters.dtci(self.meters_prefix + "rcs.cache_dtc_read", SolBase.msdiff(ms_start))
def _start_all(self, server_config): """ Start server and client. :param server_config: Server config. :return: tuple pysoltcp.tcpserver.TcpServer.TcpServer,pysoltcp.tcp_client.TcpSimpleClient.TcpSimpleClient :rtype tuple """ # Allocate self.tcp_server = TcpServer(server_config) # Check self.assertIsNotNone(self.tcp_server) self.assertFalse(self.tcp_server._is_started) self.assertTrue(self.tcp_server._server is None) # Start self.assertTrue(self.tcp_server.start_server()) self.assertTrue(self.tcp_server._is_started) self.assertFalse(self.tcp_server._server is None) # Client config client_config = TcpClientConfig() client_config.target_addr = "127.0.0.1" client_config.target_port = 3201 client_config.debug_log = True # Client self.tcp_client = TcpSimpleClient(client_config) # Check self.assertTrue(self.tcp_client.current_socket is None) self.assertTrue(not self.tcp_client.is_connected) # Connect logger.info("Starting connect()") self.assertTrue(self.tcp_client.connect()) logger.info("Starting connect() : done") # Check client self.assertIsNotNone(self.tcp_client.current_socket) self.assertTrue(self.tcp_client.is_connected) # Wait for server logger.info("TestLog : server : wait connection") dt_start = SolBase.datecurrent() while SolBase.datediff(dt_start) < self.checkTimeOut: if len(self.tcp_server._client_connected_hash) > 0: break SolBase.sleep(int(self.checkTimeOut / 100)) logger.info("TestLog : server : wait connection : done") # Check self.assertEqual(len(self.tcp_server._client_connected_hash), 1) # Ok logger.info("Started and connected, effectiveMs=%s", self.tcp_server.get_effective_controlinterval_ms()) return self.tcp_server, self.tcp_client
def test_encoder(self): """ Test """ buf = "BUF\u001B\u0BD9\U0001A10D\u1501FUB" bin_buf = SolBase.unicode_to_binary(buf, "utf-8") self.assertEqual(buf, SolBase.binary_to_unicode(bin_buf, "utf-8"))
def tearDown(self): """ Setup (called on destroy) """ SolBase.logging_init(log_level='DEBUG', force_reset=True) Meters.write_to_logger() SolBase.logging_init(log_level='INFO', force_reset=True)
def disconnect(self): """ Disconnect """ if self._soc: SolBase.safe_close_socket(self._soc) self._soc = None
def _http_basic_internal(self, force_implementation, proxy=False, https=False): """ Test """ logger.info("Starting, impl=%s", force_implementation) hc = HttpClient() for _ in range(0, 8): # Setup request hreq = HttpRequest() hreq.force_http_implementation = force_implementation if https: hreq.uri = "https://s.knock.center/static/k/k.notif.sample.png" else: # This will redirect https hreq.uri = "http://s.knock.center/static/k/k.notif.sample.png" # Http proxy if proxy: hreq.http_proxy_host = "127.0.0.1" hreq.http_proxy_port = 1180 hresp = hc.go_http(hreq) self.assertIsNotNone(hresp) self.assertIsInstance(hresp, HttpResponse) self.assertIsNotNone(hresp.http_request) self.assertEqual(id(hreq), id(hresp.http_request)) self.assertIsNotNone(hresp.elapsed_ms) self.assertIsNone(hresp.exception) if proxy and https: # Force to urllib3 self.assertEqual(hresp.http_implementation, HttpClient.HTTP_IMPL_URLLIB3) else: self.assertEqual(hresp.http_implementation, force_implementation) self.assertIsNotNone(hresp.content_length) self.assertIsNotNone(hresp.buffer) self.assertEqual(hresp.content_length, len(hresp.buffer)) self.assertGreater(len(hresp.headers), 0) self.assertIn(hresp.status_code, [200, 302, 301]) SolBase.sleep(250)
def gevent_from_pool(self, url, http_request): """ Get a gevent client from url and request :param url: geventhttpclient.url.URL :type url: geventhttpclient.url.URL :param http_request: HttpRequest :type http_request: HttpRequest :return HTTPClient :rtype HTTPClient """ # Compute key key = "{0}#{1}#{2}#{3}#{4}#{5}#{6}#{7}#{8}#{9}#".format( # host and port url.host, url.port, # Ssl url.scheme == PROTO_HTTPS, # Other dynamic stuff http_request.https_insecure, http_request.disable_ipv6, http_request.connection_timeout_ms / 1000, http_request.network_timeout_ms / 1000, http_request.http_concurrency, http_request.http_proxy_host, http_request.http_proxy_port, ) # Check if key in self._gevent_pool: SolBase.sleep(0) return self._gevent_pool[key] # Allocate (in lock) with self._gevent_locker: # Check maxed if len(self._gevent_pool) >= self._gevent_pool_max: raise Exception("gevent pool maxed, cur={0}, max={1}".format( len(self._gevent_pool), self._gevent_pool_max )) # Ok, allocate http = HTTPClient.from_url( url, insecure=http_request.https_insecure, disable_ipv6=http_request.disable_ipv6, connection_timeout=http_request.connection_timeout_ms / 1000, network_timeout=http_request.network_timeout_ms / 1000, concurrency=http_request.http_concurrency, proxy_host=http_request.http_proxy_host, proxy_port=http_request.http_proxy_port, headers={}, ) self._gevent_pool[key] = http logger.info("Started new pool for key=%s", key) SolBase.sleep(0) return http
def put(self, key, val, ttl_ms): """ Put in cache :param key: Any key :type key: str :param val: Any val :type val: bytes,str :param ttl_ms: Ttl in ms :type ttl_ms : int :return bool (true is cached) :rtype bool """ try: if not isinstance(val, (str, bytes)): raise Exception("Value must be (str, bytes)") elif not isinstance(key, str): raise Exception("Key must be (str)") # Len of items to be added item_len = len(key) + len(val) # If item len is greater than specified threshold, do nothing if self._max_bytes and item_len > self._max_single_item_bytes: Meters.aii(self.meters_prefix + "mcs.cache_put_too_big") return False # If maxed, kick one self._purge_cache(item_len) # Key tu_obj = (SolBase.mscurrent() + ttl_ms, val) # If whenever this is already in cache, we must kick it & adjust the size if key in self._hash_key: # Get tu_old = self._hash_key[key] # Kick self._safe_unhash(key) # Notify self._notify_eviction(key, tu_old[1]) # Key hash self._hash_key[key] = tu_obj # Ordered key hash self._hash_context[key] = tu_obj # Size self._current_data_bytes.increment(item_len) # Stat Meters.aii(self.meters_prefix + "mcs.cache_put") return True except Exception as e: logger.warning("Exception, ex=%s", SolBase.extostr(e)) Meters.aii(self.meters_prefix + "mcs.cache_ex")
def test_size_limit_on_data_size_first(self): """ Test. """ # Alloc self.mem_cache = MemoryCache( max_bytes=5 * 10 * 2, max_single_item_bytes=6 * 2, purge_min_bytes=5 * 5 * 2, purge_min_count=2, max_item=sys.maxsize, ) # Put 10 items for i in range(10, 20): self.mem_cache.put("key" + str(i), SolBase.unicode_to_binary("val%s" % i, "utf-8"), 60000) logger.info("Cache=%s", self.mem_cache) # Must have all of them for i in range(10, 20): self.assertEqual(self.mem_cache.get("key" + str(i)), SolBase.unicode_to_binary("val%s" % i, "utf-8")) self.assertEqual(len(self.mem_cache._hash_key), 10) self.assertEqual(self.mem_cache._current_data_bytes.get(), 5 * 10 * 2) # Then add a new one : we will over size the cache self.mem_cache.put("key" + str(99), SolBase.unicode_to_binary("val99", "utf-8"), 60000) # We must have evicted AT least : # 5*5 + 5 bytes => 5 items + the item added => 6 items # 2 items minimum # So 6 items : 10 to 15 must be evicted logger.info("Hash = %s", self.mem_cache._hash_key) for i in range(10, 16): self.assertIsNone(self.mem_cache.get("key" + str(i))) for i in range(16, 20): self.assertEqual(self.mem_cache.get("key" + str(i)), SolBase.unicode_to_binary("val%s" % i, "utf-8")) self.assertEqual(self.mem_cache.get("key" + str(99)), SolBase.unicode_to_binary("val99", "utf-8")) self.assertEqual(len(self.mem_cache._hash_key), 5) self.assertEqual(len(self.mem_cache._hash_context), 5) self.assertEqual(self.mem_cache._current_data_bytes.get(), 5 * 5 * 2) # Try add a big one this time : must not be done (over limit) self.mem_cache.put("BIGDATA", b"aaaaaaaaaaaaaaaaaaaa", 60000) self.assertIsNone(self.mem_cache.get("BIGDATA")) # Stop self.mem_cache.stop_cache() self.mem_cache = None
def test_date(self): """ Test """ dt = SolBase.datecurrent() SolBase.sleep(100) # Gevent 1.3 : this is buggy (may be related to https://github.com/gevent/gevent/issues/1227) self.assertGreaterEqual(SolBase.datediff(dt), 100) self.assertLessEqual(SolBase.datediff(dt), 200)
def setUp(self): """ Setup (called before each test) """ # Initialize asap SolBase.voodoo_init() SolBase.set_compo_name("CompoNotSet") self.assertTrue(SolBase._voodoo_initialized) self.assertTrue(SolBase._logging_initialized)
def stop_synch(self): """ Stop synch """ # Stop calls PingServerContext.stop_synch(self) # Deadlock while True: SolBase.sleep(50)
def _stop_one_client(self): """ Test """ # Check logger.info("TestLog : server : get server context") self.assertTrue(len(self.tcp_server._client_connected_hash) == 1) ctx = None for cur in self.tcp_server._client_connected_hash.values(): ctx = cur break self.assertIsNotNone(ctx) self.assertEqual(ctx.stop_synchCalled, False) self.assertEqual(ctx.stop_synch_internalCalled, False) # Kill client self.tcpClient.disconnect() # Check client self.assertTrue(self.tcpClient.current_socket is None) self.assertFalse(self.tcpClient.is_connected) # Reset client self.tcpClient = None # Server should be disconnected from client # Wait for server logger.info("TestLog : server : wait for disconnection") dt_start = SolBase.datecurrent() while SolBase.datediff(dt_start) < self.checkTimeOutMs: ok = True if len(self.tcp_server._client_connected_hash) != 0: ok = False elif not ctx.stop_synchCalled: ok = False elif not ctx.stop_synch_internalCalled: ok = False if ok: logger.info("TestLog : server : wait for disconnection : done") break else: SolBase.sleep(int(self.checkTimeOutMs / 100)) # Check logger.info("TestLog : server : check for disconnection") self.assertTrue(len(self.tcp_server._client_connected_hash) == 0) # Check self.assertEqual(ctx.stop_synchCalled, True) self.assertEqual(ctx.stop_synch_internalCalled, True)