def safe_close_socket(cls, soc_to_close): """ Safe close a socket :param cls: cls :param soc_to_close: socket :return: Nothing """ if soc_to_close is None: return try: soc_to_close.shutdown(2) except Exception as e: logger.debug("Socket shutdown ex=%s", SolBase.extostr(e)) try: soc_to_close.close() except Exception as e: logger.debug("Socket close ex=%s", SolBase.extostr(e)) try: del soc_to_close except Exception as e: logger.debug("Socket del ex=%s", SolBase.extostr(e))
def _write_to_socket(self, local_buffer): """ Write to the socket. """ # Check if not self.__is_running(): logger.debug("not connected, doing nothing, self=%s", self) return False try: # self.current_socket.sendall(local_buffer) return True except error as e: # [Errno 11] Resource temporarily unavailable # Means that nothing is available to read. # If not this, we raise. if e.args[0] != EWOULDBLOCK: # Raise :) logger.debug("Exception, ex=%s, self=%s", SolBase.extostr(e), self) self._disconnect_helper("_write_to_socket : error / No EWOULDBLOCK") return False else: # Normal logger.debug("normal exception/EWOULDBLOCK, ex=%s, self=%s", e, self) return False except Exception as e: logger.info("Exception, ex=%s, self=%s", SolBase.extostr(e), self) self._disconnect_helper("_write_to_socket : Exception") return False
def append_binary_to_file(file_name, bin_buf): """ Write to the specified filename, the provided binary buffer. Create the file if required. :param file_name: File name. :type file_name: str :param bin_buf: Binary buffer to write. :type bin_buf: bytes :return: The number of bytes written or lt 0 if error. :rtype int """ # Go rd = None try: # Open (text : open return a io.BufferedReader) rd = open(file_name, "ab+") # Read everything return rd.write(bin_buf) except IOError as e: # Exception... logger.error("append_binary_to_file : IOError, ex=%s", SolBase.extostr(e)) return -1 except Exception as e: logger.error("append_binary_to_file : Exception, ex=%s", SolBase.extostr(e)) return -1 finally: # Close if not None... if rd: rd.close()
def file_to_binary(file_name): """ Load a file toward a binary buffer. :param file_name: File name. :type file_name: str :return: Return the binary buffer or None in case of error. :rtype: bytes,None """ # Check if not FileUtility.is_file_exist(file_name): logger.error("file_to_binary : file_name not exist, file_name=%s", file_name) return None # Go rd = None try: # Open (binary : open return a io.BufferedReader) rd = open(file_name, "rb") # Read everything return rd.read() except IOError as e: # Exception... logger.error("IOError, ex=%s", SolBase.extostr(e)) return None except Exception as e: logger.error("Exception, ex=%s", SolBase.extostr(e)) return None finally: # Close if not None... if rd: rd.close()
def _daemon_stop(self): """ Stop the Daemon # Status : OK, implemented # - Running : exit 0 => OK # - Not running and pid file exist : exit 1 => OK # - Not running : exit 3 => OK # - Other : 4 => NOT TESTED """ logger.debug("entering") # Get the pid from the pidfile pid = self._get_running_pid() if not pid: logger.info("Daemon is not running, pidFile=%s", self._pidfile) return # Stop it logger.debug("sending SIGTERM, pid=%s, pidFile=%s", pid, self._pidfile) try: os.kill(pid, SIGTERM) except OSError as ex: if ex.errno == errno.ESRCH: logger.info("SIGTERM failed, ESRCH, ex=%s", SolBase.extostr(ex)) else: logger.info("SIGTERM failed, not an ESRCH, ex=%s", SolBase.extostr(ex)) except Exception as ex: logger.info("SIGTERM failed, not an OSError, going exit(1), ex=%s", SolBase.extostr(ex)) sys.exit(1) finally: if os.path.exists(self._pidfile): logger.debug("Removing pidFile=%s", self._pidfile) self._remove_pid_file() # Ok logger.debug("SIGTERM sent") ms_start = SolBase.mscurrent() # Validate proc_target = "/proc/%d" % pid while SolBase.msdiff(ms_start) < self._timeout_ms: if os.path.exists(proc_target): SolBase.sleep(10) continue # Over logger.info("SIGTERM success, pid=%s", pid) self._remove_pid_file() return # Not cool logger.warning("SIGTERM timeout=%s ms, pid=%s", self._timeout_ms, pid)
def _watchdog_run(self): """ Watch dog :return Nothing """ if not self._is_started: return reschedule = True try: # Current meters Meters.ai(self.meters_prefix + "mcs.cur_bytes").set( self._current_data_bytes.get()) Meters.ai(self.meters_prefix + "mcs.cur_size_hash").set( len(self._hash_key)) # Evict ms = SolBase.mscurrent() evicted_count = self._evict_all_expired_keys() # Check (evict can take some time) if not self._is_started: return Meters.dtci(self.meters_prefix + "mcs.cache_dtc_watchdog", SolBase.msdiff(ms)) # Stat if evicted_count > 0: Meters.aii(self.meters_prefix + "mcs.cache_evict_ttl_watchdog", evicted_count) # Callback (unittest) if self._cb_watchdog: reschedule = self._cb_watchdog(evicted_count) except Exception as e: if self._is_started: logger.error("_watchdog_run : Exception, id=%s, e=%s", id(self), SolBase.extostr(e)) Meters.aii(self.meters_prefix + "mcs.cache_ex") else: logger.debug("_watchdog_run : Exception, id=%s, e=%s", id(self), SolBase.extostr(e)) reschedule = False finally: Meters.aii(self.meters_prefix + "mcs.cache_watchdog_run_count") # Schedule next write if reschedule and self._is_started: self._schedule_next_watchdog()
def setUp(self): """ Setup """ MysqlApi.reset_pools() Meters.reset() d_conf_root = { "host": "localhost", "port": 3306, "database": None, "user": "******", "password": "******", "autocommit": True, } # exec_n try: ar = MysqlApi.exec_n(d_conf_root, "DROP DATABASE IF EXISTS pysolmysql_test;") logger.info("ar=%s", ar) except Exception as e: logger.debug("Ex=%s", SolBase.extostr(e)) MysqlApi.exec_n(d_conf_root, "CREATE DATABASE IF NOT EXISTS pysolmysql_test;") # Full reset MysqlApi.reset_pools() Meters.reset()
def handle(self, data, address): """ Handle one udp message :param data: data :type data: str :param address: address :type address: str """ ms_start = SolBase.mscurrent() try: # Handle data pass # Stats Meters.aii("resolvusclient_udp_recv") except Exception as e: # Log logger.warning( "Handle failed, data_len=%s, address=%s, data=%s, ex=%s", len(data), repr(address), repr(data), SolBase.extostr(e)) # Stat Meters.aii("resolvusclient_udp_recv_ex") finally: Meters.dtci("resolvusclient_udp_recv_dtc", SolBase.msdiff(ms_start))
def stop_cache(self): """ Stop the cache :return """ with self._run_lock: try: # Check if not self._is_started: return # Signal self._is_started = False # Kill greenlet if self._watchdog_greenlet: self._watchdog_greenlet.kill(block=True) self._watchdog_greenlet = None except Exception as e: logger.error("Exception, ex=%s", SolBase.extostr(e)) finally: # Reset self._is_started = False self._watchdog_greenlet = None
def stop(self): """ Stop """ # Signal out of lock (may help greenlet to exit itself) self._is_running = False with self._locker: try: lifecyclelogger.info("Stop : stopping") # Stop if self._wsgi_server: self._wsgi_server.close() self._wsgi_server = None # Kill the greenlet if self._server_greenlet: logger.info("_server_greenlet.kill") self._server_greenlet.kill() logger.info("_server_greenlet.kill done") # gevent.kill(self._server_greenlet) self._server_greenlet.join() self._server_greenlet = None lifecyclelogger.info("Stop : stopped") except Exception as e: logger.error("Exception, e=%s", SolBase.extostr(e))
def test_tcp_svr_start_cli_connect_svrstop_onstoptrue(self): """ Test """ # Instance self.tcp_server = None try: # Start server self.tcp_server = self._start_server_and_check() # Force not call self.tcp_server._tcp_server_config.onstop_call_client_stopsynch = True # Start client, check and stop client self._start_one_client() # Stop server self._stop_server_and_check_withoneclientconnected() except Exception as e: logger.error("Exception in test, ex=%s", SolBase.extostr(e)) raise finally: if self.tcp_server: self.tcp_server.stop_server() if self.tcpClient: self.tcpClient.disconnect()
def test_tcp_svr_start_cli_connectallpingcheck_ssl(self): """ Test """ # Instance self.tcp_server = None # ssl self.testSsl = True try: # Start server self._start_server_and_check() # Start client, check and stop client self._start_one_client_checkallping_stop() # Stop server self._stop_server_and_check() except Exception as e: logger.error("Exception in test, ex=%s", SolBase.extostr(e)) raise finally: if self.tcp_server: self.tcp_server.stop_server() if self.tcpClient: self.tcpClient.disconnect()
def start(self): """ Start """ with self._locker: try: lifecyclelogger.info("Start : starting") # Check if self._is_running: logger.warning("Already running, doing nothing") # Start self._server_greenlet = gevent.spawn(self._server_forever) SolBase.sleep(0) # Wait lifecyclelogger.debug("Start : waiting") self._start_event.wait() SolBase.sleep(0) # Signal self._is_running = True lifecyclelogger.info("Start : started") except Exception as e: logger.error("Exception, e=%s", SolBase.extostr(e))
def _connection_ping(self, conn): """ Ping connection This send a ping, write/read toward mysql. :param conn: pymysql.connections.Connection :type conn: pymysql.connections.Connection :return bool :rtype bool """ Meters.aii("k.db_pool.mysql.call._connection_ping") # noinspection PyBroadException try: # TODO : PING MODE MUST BE CONFIGURABLE conn.ping(reconnect=False) except Exception as e: Meters.aii("k.db_pool.mysql.ex_ping") logger.debug("Ping failed, obj=%s, ex=%s", conn, SolBase.extostr(e)) return False else: return True
def generate_server_keys(cls): """ Generate server keys :return: Path to server keys. """ try: # Get current dir current_dir = dirname( abspath(__file__)) + SolBase.get_pathseparator() # Certificates path cert_path = current_dir + "Certificates" + SolBase.get_pathseparator( ) # Copy to /tmp (required for some files) shutil.copyfile(cert_path + "server.key", "/tmp/server.key") shutil.copyfile(cert_path + "server.crt", "/tmp/server.crt") # Ok return cert_path except Exception as e: logger.error("generate_server_keys : Exception, ex=%s", SolBase.extostr(e)) return None
def _redis_open(cls, d_param): """ Open a redis instance :param d_param: dict :type d_param: dict :return tuple pool, redis_instance :rtype tuple """ # Pool try: pool = redis.ConnectionPool( host=d_param["host"], port=d_param["port"], db=d_param["db"], max_connections=d_param["max_connections"]) logger.info("Initialized pool=%s, d_param=%s", pool, d_param) # Redis redis_instance = redis.Redis(connection_pool=pool) logger.info("Initialized redis_instance=%s, d_param=%s", redis_instance, d_param) return pool, redis_instance except Exception as e: logger.error("Exception, ex=%s", SolBase.extostr(e)) raise
def stop(self, timeout=None): """ Stop """ if not self._is_started: logger.warn("Not started, bypass") return # Base stop logger.info("Stopping base") DatagramServer.stop(self, timeout=timeout) logger.info("Stopped base") # Greenlet stop if self._server_greenlet: logger.info("Killing _server_greenlet") self._server_greenlet.kill() self._server_greenlet = None # Close socket SocketHelpers.safe_close_socket(self._soc) # Remove socket try: if os.path.exists(self._socket_name): os.remove(self._socket_name) except Exception as e: logger.warn("Socket file remove ex=%s", SolBase.extostr(e)) # Signal stopped self._is_started = False
def _schedule_next_watchdog(self): """ Schedule next run. """ try: # Started ? if not self._is_started: return with self._run_lock: # Re-check if not self._is_started: return # Yes, schedule self._watchdog_greenlet = gevent.spawn_later( self._watchdog_interval_ms * 0.001, self._watchdog_run) # Wait SolBase.sleep(0) except Exception as e: logger.error("_schedule_next_watchdog : Exception, e=%s", SolBase.extostr(e)) finally: pass
def _server_forever(self): """ Exec loop """ try: # Alloc logger.info("Allocating WSGIServer") self._wsgi_server = WSGIServer(listener=('localhost', 7900), application=self.on_request) logger.info("Starting, %s, %s", self._wsgi_server.address, _parse_address(self._wsgi_server.address)) SolBase.sleep(0) # Signal logger.info("Signaling _start_event") self._start_event.set() SolBase.sleep(0) # This will block until signaled logger.info("Calling serve_forever") self._wsgi_server.serve_forever() except Exception as e: logger.error("Ex=%s", SolBase.extostr(e)) # This is fatal, we exit, we cannot serve exit(-1) finally: logger.info("Clearing _start_event") self._start_event.clear()
def get(self, key): """ Get from cache. :param key: Any key :type key: str :return An obj or null if not in cache :rtype bytes, None """ ms_start = SolBase.mscurrent() try: if not isinstance(key, (bytes, str)): raise Exception("Key must be (bytes, str)") # Use read redis v = self._read_redis.get(key) if v: Meters.aii(self.meters_prefix + "rcs.cache_get_hit") else: Meters.aii(self.meters_prefix + "rcs.cache_get_miss") return v except Exception as e: logger.warning("Exception, ex=%s", SolBase.extostr(e)) Meters.aii(self.meters_prefix + "rcs.cache_ex") return None finally: Meters.dtci(self.meters_prefix + "rcs.cache_dtc_read", SolBase.msdiff(ms_start))
def put(self, key, val, ttl_ms): """ Put in cache :param key: Any key :type key: str :param val: Any val :type val: bytes,str :param ttl_ms: Ttl in ms :type ttl_ms : int :return bool (true is cached) :rtype bool """ try: if not isinstance(val, (str, bytes)): raise Exception("Value must be (str, bytes)") elif not isinstance(key, str): raise Exception("Key must be (str)") # Len of items to be added item_len = len(key) + len(val) # If item len is greater than specified threshold, do nothing if self._max_bytes and item_len > self._max_single_item_bytes: Meters.aii(self.meters_prefix + "mcs.cache_put_too_big") return False # If maxed, kick one self._purge_cache(item_len) # Key tu_obj = (SolBase.mscurrent() + ttl_ms, val) # If whenever this is already in cache, we must kick it & adjust the size if key in self._hash_key: # Get tu_old = self._hash_key[key] # Kick self._safe_unhash(key) # Notify self._notify_eviction(key, tu_old[1]) # Key hash self._hash_key[key] = tu_obj # Ordered key hash self._hash_context[key] = tu_obj # Size self._current_data_bytes.increment(item_len) # Stat Meters.aii(self.meters_prefix + "mcs.cache_put") return True except Exception as e: logger.warning("Exception, ex=%s", SolBase.extostr(e)) Meters.aii(self.meters_prefix + "mcs.cache_ex")
def tearDown(self): """ Setup (called on destroy) """ try: if self.tcp_server: self.tcp_server.stop_server() except Exception as e: self.fail("Exception on stop_server, ex=" + SolBase.extostr(e)) try: if self.tcpClient: self.tcpClient.disconnect() except Exception as e: self.fail("Exception on disconnect, ex=" + SolBase.extostr(e)) Utility.test_wait()
def _internal_test(self, client_count, ping_interval_ms, my_run_time_ms=20000): """ Test. :param client_count: Client count. :param ping_interval_ms: Ping interval in millis. :param my_run_time_ms: Run time in millis. :return: nothing. """ try: # Overrides statics (beuuurk but so gooood) self.clientMaxCount = client_count self.serverPingIntervalMs = ping_interval_ms self.clientPingIntervalMs = ping_interval_ms self.runTimeMs = my_run_time_ms # Expected server and client ping/src self.expectedPps = (float(self.clientMaxCount) / float(self.serverPingIntervalMs)) * 1000.0 logger.info("starting, client=%s, ping.ms=%s, expected.pps=%s", client_count, ping_interval_ms, self.expectedPps) # Instance self.tcp_server = None try: # Start server self._start_server_and_check() # Start client, check and stop client self._start_multi_client_checkallping_stop() # Stop server self._stop_server_and_check() except Exception as e: logger.error("Exception in test, ex=%s", SolBase.extostr(e)) raise finally: if self.tcp_server: self.tcp_server.stop_server() finally: # Final stats logger.info("Finished, final stats bellow") Meters.write_to_logger() # Reset self.tcp_server = None # Sleep Utility.test_wait()
def append_text_to_file(file_name, text_buffer, encoding, overwrite=False): """ Write to the specified filename, the provided binary buffer Create the file if required. :param file_name: File name. :type file_name: str :param text_buffer: Text buffer to write. :type text_buffer: str :param encoding: The encoding to use. :type encoding: str :param overwrite: If true, file is overwritten. :type overwrite: bool :return: The number of bytes written or lt 0 if error. :rtype int """ # Go rd = None try: # Open (text : open return a io.BufferedReader) if not overwrite: rd = codecs.open(file_name, "a+", encoding, "strict", -1) else: rd = codecs.open(file_name, "w", encoding, "strict", -1) # Read everything # CAUTION : 2.7 return None :( return rd.write(text_buffer) except IOError as e: # Exception... logger.error("append_text_to_file : IOError, ex=%s", SolBase.extostr(e)) return -1 except Exception as e: logger.error("append_text_to_file : Exception, ex=%s", SolBase.extostr(e)) return -1 finally: # Close if not None... if rd: rd.close()
def is_os_64(cls): """ Return true is platform is 64 bits :return bool :rtype bool """ try: return platform.machine().find('64') >= 0 except Exception as e: # Not normal, but fallback 64 bits logger.warn("Ex=%s", SolBase.extostr(e)) return True
def connect(self, socket_name): """ Connect (not available on windows) :param socket_name: str :type socket_name: str """ try: self._soc = socket.socket(socket.AF_UNIX, type=socket.SOCK_DGRAM) self._soc.connect(socket_name) except Exception as e: logger.warning("connect failed, ex=%s", SolBase.extostr(e)) raise
def file_to_textbuffer(file_name, encoding): """ Load a file toward a text buffer (UTF-8), using the specify encoding while reading. CAUTION : This will read the whole file IN MEMORY. :param file_name: File name. :type file_name: str :param encoding: Encoding to use. :type encoding: str :return: A text buffer or None in case of error. :rtype str """ # Check if not FileUtility.is_file_exist(file_name): logger.error( "file_to_textbuffer : file_name not exist, file_name=%s", file_name) return None # Go rd = None try: # Open (text : open return a io.BufferedReader) rd = codecs.open(file_name, "r", encoding, "strict", -1) # Read everything return rd.read() except IOError as e: # Exception... logger.error("file_to_binary : IOError, ex=%s", SolBase.extostr(e)) return None except Exception as e: logger.error("file_to_binary : Exception, ex=%s", SolBase.extostr(e)) return None finally: # Close if not None... if rd: rd.close()
def _remove_client_stop_business(self, old_client, evt): """ Remove internal :param old_client: oldclient :type old_client: pysoltcp.tcpserver.clientcontext.TcpServerClientContext.TcpServerClientContext :param evt: gevent.Event :type evt: gevent.Event """ # ------------------------- # Stop the client (Business call here) # We do NOT call if : # - service is stopping AND onstop_call_client_stopsynch==False # ------------------------- try: cid = old_client.get_client_id() logger.debug("_remove_client_stop_business call, cid=%s", cid) if self._is_running: # ------------------- # Running, call # ------------------- logger.debug("stop_synch call (_is_running==%s), cid=%s", self._is_running, cid) old_client.stop_synch() elif not self._is_running and self._tcp_server_config.onstop_call_client_stopsynch: # ------------------- # Not running + call ON : call # ------------------- logger.debug( "stop_synch call (_is_running==%s + onstop_call_client_stopsynch==%s), cid=%s", self._is_running, self._tcp_server_config.onstop_call_client_stopsynch, cid) old_client.stop_synch() else: # ------------------- # No call # ------------------- logger.debug( "stop_synch NOT CALLED (_is_running==%s + onstop_call_client_stopsynch==%s), cid=%s", self._is_running, self._tcp_server_config.onstop_call_client_stopsynch, cid) pass except Exception as e: logger.warning("Ex=%s", SolBase.extostr(e)) finally: evt.set()
def tearDown(self): """ Setup (called on destroy) """ if not SolBase.get_master_process(): return logger.info("tearDown : entering") try: if self.tcp_server: self.tcp_server.stop_server() except Exception as e: self.fail("Exception on stop_server, ex=" + SolBase.extostr(e)) try: if self.tcp_client: self.tcp_client.disconnect() except Exception as e: self.fail("Exception on disconnect, ex=" + SolBase.extostr(e)) Utility.test_wait()
def _write_loop(self): """ High level read loop on socket """ logger.debug("entering now, self=%s", self) try: self._write_loop_internal() except GreenletExit: logger.debug("exiting due to GreenletExit, self=%s", self) return except Exception as e: logger.error("Exception raised, ex=%s, self=%s", SolBase.extostr(e), self) finally: logger.debug("exiting now, , self=%s", self) SolBase.sleep(0)