def run(hs): PROFILE_SYNAPSE = False if PROFILE_SYNAPSE: def profile(func): from cProfile import Profile from threading import current_thread def profiled(*args, **kargs): profile = Profile() profile.enable() func(*args, **kargs) profile.disable() ident = current_thread().ident profile.dump_stats("/tmp/%s.%s.%i.pstat" % (hs.hostname, func.__name__, ident)) return profiled from twisted.python.threadpool import ThreadPool ThreadPool._worker = profile(ThreadPool._worker) reactor.run = profile(reactor.run) _base.start_reactor( "synapse-homeserver", soft_file_limit=hs.config.soft_file_limit, gc_thresholds=hs.config.gc_thresholds, pid_file=hs.config.pid_file, daemonize=hs.config.daemonize, print_pidfile=hs.config.print_pidfile, logger=logger, )
def run(hs: HomeServer) -> None: _base.start_reactor( "synapse-homeserver", soft_file_limit=hs.config.server.soft_file_limit, gc_thresholds=hs.config.server.gc_thresholds, pid_file=hs.config.server.pid_file, daemonize=hs.config.server.daemonize, print_pidfile=hs.config.server.print_pidfile, logger=logger, )
def run(hs): PROFILE_SYNAPSE = False if PROFILE_SYNAPSE: def profile(func): from cProfile import Profile from threading import current_thread def profiled(*args, **kargs): profile = Profile() profile.enable() func(*args, **kargs) profile.disable() ident = current_thread().ident profile.dump_stats("/tmp/%s.%s.%i.pstat" % (hs.hostname, func.__name__, ident)) return profiled from twisted.python.threadpool import ThreadPool ThreadPool._worker = profile(ThreadPool._worker) reactor.run = profile(reactor.run) clock = hs.get_clock() start_time = clock.time() stats = {} # Contains the list of processes we will be monitoring # currently either 0 or 1 stats_process = [] def start_phone_stats_home(): return run_as_background_process("phone_stats_home", phone_stats_home) @defer.inlineCallbacks def phone_stats_home(): logger.info("Gathering stats for reporting") now = int(hs.get_clock().time()) uptime = int(now - start_time) if uptime < 0: uptime = 0 stats["homeserver"] = hs.config.server_name stats["timestamp"] = now stats["uptime_seconds"] = uptime version = sys.version_info stats["python_version"] = "{}.{}.{}".format(version.major, version.minor, version.micro) stats["total_users"] = yield hs.get_datastore().count_all_users() total_nonbridged_users = yield hs.get_datastore( ).count_nonbridged_users() stats["total_nonbridged_users"] = total_nonbridged_users daily_user_type_results = yield hs.get_datastore( ).count_daily_user_type() for name, count in iteritems(daily_user_type_results): stats["daily_user_type_" + name] = count room_count = yield hs.get_datastore().get_room_count() stats["total_room_count"] = room_count stats["daily_active_users"] = yield hs.get_datastore( ).count_daily_users() stats["daily_active_rooms"] = yield hs.get_datastore( ).count_daily_active_rooms() stats["daily_messages"] = yield hs.get_datastore( ).count_daily_messages() r30_results = yield hs.get_datastore().count_r30_users() for name, count in iteritems(r30_results): stats["r30_users_" + name] = count daily_sent_messages = yield hs.get_datastore( ).count_daily_sent_messages() stats["daily_sent_messages"] = daily_sent_messages stats["cache_factor"] = CACHE_SIZE_FACTOR stats["event_cache_size"] = hs.config.event_cache_size if len(stats_process) > 0: stats["memory_rss"] = 0 stats["cpu_average"] = 0 for process in stats_process: stats["memory_rss"] += process.memory_info().rss stats["cpu_average"] += int(process.cpu_percent(interval=None)) stats["database_engine"] = hs.get_datastore().database_engine_name stats["database_server_version"] = hs.get_datastore( ).get_server_version() logger.info("Reporting stats to matrix.org: %s" % (stats, )) try: yield hs.get_simple_http_client().put_json( "https://matrix.org/report-usage-stats/push", stats) except Exception as e: logger.warn("Error reporting stats: %s", e) def performance_stats_init(): try: process = psutil.Process() # Ensure we can fetch both, and make the initial request for cpu_percent # so the next request will use this as the initial point. process.memory_info().rss process.cpu_percent(interval=None) logger.info("report_stats can use psutil") stats_process.append(process) except (AttributeError): logger.warning( "Unable to read memory/cpu stats. Disabling reporting.") def generate_user_daily_visit_stats(): return run_as_background_process( "generate_user_daily_visits", hs.get_datastore().generate_user_daily_visits, ) # Rather than update on per session basis, batch up the requests. # If you increase the loop period, the accuracy of user_daily_visits # table will decrease clock.looping_call(generate_user_daily_visit_stats, 5 * 60 * 1000) # monthly active user limiting functionality def reap_monthly_active_users(): return run_as_background_process( "reap_monthly_active_users", hs.get_datastore().reap_monthly_active_users, ) clock.looping_call(reap_monthly_active_users, 1000 * 60 * 60) reap_monthly_active_users() @defer.inlineCallbacks def generate_monthly_active_users(): current_mau_count = 0 reserved_count = 0 store = hs.get_datastore() if hs.config.limit_usage_by_mau or hs.config.mau_stats_only: current_mau_count = yield store.get_monthly_active_count() reserved_count = yield store.get_registered_reserved_users_count() current_mau_gauge.set(float(current_mau_count)) registered_reserved_users_mau_gauge.set(float(reserved_count)) max_mau_gauge.set(float(hs.config.max_mau_value)) def start_generate_monthly_active_users(): return run_as_background_process( "generate_monthly_active_users", generate_monthly_active_users, ) start_generate_monthly_active_users() if hs.config.limit_usage_by_mau or hs.config.mau_stats_only: clock.looping_call(start_generate_monthly_active_users, 5 * 60 * 1000) # End of monthly active user settings if hs.config.report_stats: logger.info("Scheduling stats reporting for 3 hour intervals") clock.looping_call(start_phone_stats_home, 3 * 60 * 60 * 1000) # We need to defer this init for the cases that we daemonize # otherwise the process ID we get is that of the non-daemon process clock.call_later(0, performance_stats_init) # We wait 5 minutes to send the first set of stats as the server can # be quite busy the first few minutes clock.call_later(5 * 60, start_phone_stats_home) if hs.config.daemonize and hs.config.print_pidfile: print(hs.config.pid_file) _base.start_reactor( "synapse-homeserver", hs.config.soft_file_limit, hs.config.gc_thresholds, hs.config.pid_file, hs.config.daemonize, hs.config.cpu_affinity, logger, )
def run(hs): PROFILE_SYNAPSE = False if PROFILE_SYNAPSE: def profile(func): from cProfile import Profile from threading import current_thread def profiled(*args, **kargs): profile = Profile() profile.enable() func(*args, **kargs) profile.disable() ident = current_thread().ident profile.dump_stats("/tmp/%s.%s.%i.pstat" % (hs.hostname, func.__name__, ident)) return profiled from twisted.python.threadpool import ThreadPool ThreadPool._worker = profile(ThreadPool._worker) reactor.run = profile(reactor.run) clock = hs.get_clock() start_time = clock.time() stats = {} @defer.inlineCallbacks def phone_stats_home(): logger.info("Gathering stats for reporting") now = int(hs.get_clock().time()) uptime = int(now - start_time) if uptime < 0: uptime = 0 stats["homeserver"] = hs.config.server_name stats["timestamp"] = now stats["uptime_seconds"] = uptime stats["total_users"] = yield hs.get_datastore().count_all_users() total_nonbridged_users = yield hs.get_datastore( ).count_nonbridged_users() stats["total_nonbridged_users"] = total_nonbridged_users room_count = yield hs.get_datastore().get_room_count() stats["total_room_count"] = room_count stats["daily_active_users"] = yield hs.get_datastore( ).count_daily_users() stats["daily_active_rooms"] = yield hs.get_datastore( ).count_daily_active_rooms() stats["daily_messages"] = yield hs.get_datastore( ).count_daily_messages() daily_sent_messages = yield hs.get_datastore( ).count_daily_sent_messages() stats["daily_sent_messages"] = daily_sent_messages logger.info("Reporting stats to matrix.org: %s" % (stats, )) try: yield hs.get_simple_http_client().put_json( "https://matrix.org/report-usage-stats/push", stats) except Exception as e: logger.warn("Error reporting stats: %s", e) if hs.config.report_stats: logger.info("Scheduling stats reporting for 3 hour intervals") clock.looping_call(phone_stats_home, 3 * 60 * 60 * 1000) # We wait 5 minutes to send the first set of stats as the server can # be quite busy the first few minutes clock.call_later(5 * 60, phone_stats_home) if hs.config.daemonize and hs.config.print_pidfile: print(hs.config.pid_file) _base.start_reactor( "synapse-homeserver", hs.config.soft_file_limit, hs.config.gc_thresholds, hs.config.pid_file, hs.config.daemonize, hs.config.cpu_affinity, logger, )
def run(hs): PROFILE_SYNAPSE = False if PROFILE_SYNAPSE: def profile(func): from cProfile import Profile from threading import current_thread def profiled(*args, **kargs): profile = Profile() profile.enable() func(*args, **kargs) profile.disable() ident = current_thread().ident profile.dump_stats( "/tmp/%s.%s.%i.pstat" % (hs.hostname, func.__name__, ident) ) return profiled from twisted.python.threadpool import ThreadPool ThreadPool._worker = profile(ThreadPool._worker) reactor.run = profile(reactor.run) clock = hs.get_clock() stats = {} def performance_stats_init(): _stats_process.clear() _stats_process.append( (int(hs.get_clock().time()), resource.getrusage(resource.RUSAGE_SELF)) ) def start_phone_stats_home(): return run_as_background_process( "phone_stats_home", phone_stats_home, hs, stats ) def generate_user_daily_visit_stats(): return run_as_background_process( "generate_user_daily_visits", hs.get_datastore().generate_user_daily_visits ) # Rather than update on per session basis, batch up the requests. # If you increase the loop period, the accuracy of user_daily_visits # table will decrease clock.looping_call(generate_user_daily_visit_stats, 5 * 60 * 1000) # monthly active user limiting functionality def reap_monthly_active_users(): return run_as_background_process( "reap_monthly_active_users", hs.get_datastore().reap_monthly_active_users ) clock.looping_call(reap_monthly_active_users, 1000 * 60 * 60) reap_monthly_active_users() @defer.inlineCallbacks def generate_monthly_active_users(): current_mau_count = 0 reserved_users = () store = hs.get_datastore() if hs.config.limit_usage_by_mau or hs.config.mau_stats_only: current_mau_count = yield store.get_monthly_active_count() reserved_users = yield store.get_registered_reserved_users() current_mau_gauge.set(float(current_mau_count)) registered_reserved_users_mau_gauge.set(float(len(reserved_users))) max_mau_gauge.set(float(hs.config.max_mau_value)) def start_generate_monthly_active_users(): return run_as_background_process( "generate_monthly_active_users", generate_monthly_active_users ) start_generate_monthly_active_users() if hs.config.limit_usage_by_mau or hs.config.mau_stats_only: clock.looping_call(start_generate_monthly_active_users, 5 * 60 * 1000) # End of monthly active user settings if hs.config.report_stats: logger.info("Scheduling stats reporting for 3 hour intervals") clock.looping_call(start_phone_stats_home, 3 * 60 * 60 * 1000) # We need to defer this init for the cases that we daemonize # otherwise the process ID we get is that of the non-daemon process clock.call_later(0, performance_stats_init) # We wait 5 minutes to send the first set of stats as the server can # be quite busy the first few minutes clock.call_later(5 * 60, start_phone_stats_home) _base.start_reactor( "synapse-homeserver", soft_file_limit=hs.config.soft_file_limit, gc_thresholds=hs.config.gc_thresholds, pid_file=hs.config.pid_file, daemonize=hs.config.daemonize, print_pidfile=hs.config.print_pidfile, logger=logger, )
def run(hs): PROFILE_SYNAPSE = False if PROFILE_SYNAPSE: def profile(func): from cProfile import Profile from threading import current_thread def profiled(*args, **kargs): profile = Profile() profile.enable() func(*args, **kargs) profile.disable() ident = current_thread().ident profile.dump_stats("/tmp/%s.%s.%i.pstat" % (hs.hostname, func.__name__, ident)) return profiled from twisted.python.threadpool import ThreadPool ThreadPool._worker = profile(ThreadPool._worker) reactor.run = profile(reactor.run) clock = hs.get_clock() start_time = clock.time() stats = {} # Contains the list of processes we will be monitoring # currently either 0 or 1 stats_process = [] @defer.inlineCallbacks def phone_stats_home(): logger.info("Gathering stats for reporting") now = int(hs.get_clock().time()) uptime = int(now - start_time) if uptime < 0: uptime = 0 stats["homeserver"] = hs.config.server_name stats["timestamp"] = now stats["uptime_seconds"] = uptime stats["total_users"] = yield hs.get_datastore().count_all_users() total_nonbridged_users = yield hs.get_datastore( ).count_nonbridged_users() stats["total_nonbridged_users"] = total_nonbridged_users room_count = yield hs.get_datastore().get_room_count() stats["total_room_count"] = room_count stats["daily_active_users"] = yield hs.get_datastore( ).count_daily_users() stats["daily_active_rooms"] = yield hs.get_datastore( ).count_daily_active_rooms() stats["daily_messages"] = yield hs.get_datastore( ).count_daily_messages() r30_results = yield hs.get_datastore().count_r30_users() for name, count in r30_results.iteritems(): stats["r30_users_" + name] = count daily_sent_messages = yield hs.get_datastore( ).count_daily_sent_messages() stats["daily_sent_messages"] = daily_sent_messages stats["cache_factor"] = CACHE_SIZE_FACTOR stats["event_cache_size"] = hs.config.event_cache_size if len(stats_process) > 0: stats["memory_rss"] = 0 stats["cpu_average"] = 0 for process in stats_process: stats["memory_rss"] += process.memory_info().rss stats["cpu_average"] += int(process.cpu_percent(interval=None)) logger.info("Reporting stats to matrix.org: %s" % (stats, )) try: yield hs.get_simple_http_client().put_json( "https://matrix.org/report-usage-stats/push", stats) except Exception as e: logger.warn("Error reporting stats: %s", e) def performance_stats_init(): try: import psutil process = psutil.Process() # Ensure we can fetch both, and make the initial request for cpu_percent # so the next request will use this as the initial point. process.memory_info().rss process.cpu_percent(interval=None) logger.info("report_stats can use psutil") stats_process.append(process) except (ImportError, AttributeError): logger.warn( "report_stats enabled but psutil is not installed or incorrect version." " Disabling reporting of memory/cpu stats." " Ensuring psutil is available will help matrix.org track performance" " changes across releases.") if hs.config.report_stats: logger.info("Scheduling stats reporting for 3 hour intervals") clock.looping_call(phone_stats_home, 3 * 60 * 60 * 1000) # We need to defer this init for the cases that we daemonize # otherwise the process ID we get is that of the non-daemon process clock.call_later(0, performance_stats_init) # We wait 5 minutes to send the first set of stats as the server can # be quite busy the first few minutes clock.call_later(5 * 60, phone_stats_home) if hs.config.daemonize and hs.config.print_pidfile: print(hs.config.pid_file) _base.start_reactor( "synapse-homeserver", hs.config.soft_file_limit, hs.config.gc_thresholds, hs.config.pid_file, hs.config.daemonize, hs.config.cpu_affinity, logger, )
def run(hs): PROFILE_SYNAPSE = False if PROFILE_SYNAPSE: def profile(func): from cProfile import Profile from threading import current_thread def profiled(*args, **kargs): profile = Profile() profile.enable() func(*args, **kargs) profile.disable() ident = current_thread().ident profile.dump_stats("/tmp/%s.%s.%i.pstat" % ( hs.hostname, func.__name__, ident )) return profiled from twisted.python.threadpool import ThreadPool ThreadPool._worker = profile(ThreadPool._worker) reactor.run = profile(reactor.run) clock = hs.get_clock() start_time = clock.time() stats = {} # Contains the list of processes we will be monitoring # currently either 0 or 1 stats_process = [] def start_phone_stats_home(): return run_as_background_process("phone_stats_home", phone_stats_home) @defer.inlineCallbacks def phone_stats_home(): logger.info("Gathering stats for reporting") now = int(hs.get_clock().time()) uptime = int(now - start_time) if uptime < 0: uptime = 0 stats["homeserver"] = hs.config.server_name stats["timestamp"] = now stats["uptime_seconds"] = uptime version = sys.version_info stats["python_version"] = "{}.{}.{}".format( version.major, version.minor, version.micro ) stats["total_users"] = yield hs.get_datastore().count_all_users() total_nonbridged_users = yield hs.get_datastore().count_nonbridged_users() stats["total_nonbridged_users"] = total_nonbridged_users daily_user_type_results = yield hs.get_datastore().count_daily_user_type() for name, count in iteritems(daily_user_type_results): stats["daily_user_type_" + name] = count room_count = yield hs.get_datastore().get_room_count() stats["total_room_count"] = room_count stats["daily_active_users"] = yield hs.get_datastore().count_daily_users() stats["daily_active_rooms"] = yield hs.get_datastore().count_daily_active_rooms() stats["daily_messages"] = yield hs.get_datastore().count_daily_messages() r30_results = yield hs.get_datastore().count_r30_users() for name, count in iteritems(r30_results): stats["r30_users_" + name] = count daily_sent_messages = yield hs.get_datastore().count_daily_sent_messages() stats["daily_sent_messages"] = daily_sent_messages stats["cache_factor"] = CACHE_SIZE_FACTOR stats["event_cache_size"] = hs.config.event_cache_size if len(stats_process) > 0: stats["memory_rss"] = 0 stats["cpu_average"] = 0 for process in stats_process: stats["memory_rss"] += process.memory_info().rss stats["cpu_average"] += int(process.cpu_percent(interval=None)) logger.info("Reporting stats to matrix.org: %s" % (stats,)) try: yield hs.get_simple_http_client().put_json( "https://matrix.org/report-usage-stats/push", stats ) except Exception as e: logger.warn("Error reporting stats: %s", e) def performance_stats_init(): try: process = psutil.Process() # Ensure we can fetch both, and make the initial request for cpu_percent # so the next request will use this as the initial point. process.memory_info().rss process.cpu_percent(interval=None) logger.info("report_stats can use psutil") stats_process.append(process) except (AttributeError): logger.warning( "Unable to read memory/cpu stats. Disabling reporting." ) def generate_user_daily_visit_stats(): return run_as_background_process( "generate_user_daily_visits", hs.get_datastore().generate_user_daily_visits, ) # Rather than update on per session basis, batch up the requests. # If you increase the loop period, the accuracy of user_daily_visits # table will decrease clock.looping_call(generate_user_daily_visit_stats, 5 * 60 * 1000) # monthly active user limiting functionality def reap_monthly_active_users(): return run_as_background_process( "reap_monthly_active_users", hs.get_datastore().reap_monthly_active_users, ) clock.looping_call(reap_monthly_active_users, 1000 * 60 * 60) reap_monthly_active_users() @defer.inlineCallbacks def generate_monthly_active_users(): current_mau_count = 0 reserved_count = 0 store = hs.get_datastore() if hs.config.limit_usage_by_mau: current_mau_count = yield store.get_monthly_active_count() reserved_count = yield store.get_registered_reserved_users_count() current_mau_gauge.set(float(current_mau_count)) registered_reserved_users_mau_gauge.set(float(reserved_count)) max_mau_gauge.set(float(hs.config.max_mau_value)) def start_generate_monthly_active_users(): return run_as_background_process( "generate_monthly_active_users", generate_monthly_active_users, ) start_generate_monthly_active_users() if hs.config.limit_usage_by_mau: clock.looping_call(start_generate_monthly_active_users, 5 * 60 * 1000) # End of monthly active user settings if hs.config.report_stats: logger.info("Scheduling stats reporting for 3 hour intervals") clock.looping_call(start_phone_stats_home, 3 * 60 * 60 * 1000) # We need to defer this init for the cases that we daemonize # otherwise the process ID we get is that of the non-daemon process clock.call_later(0, performance_stats_init) # We wait 5 minutes to send the first set of stats as the server can # be quite busy the first few minutes clock.call_later(5 * 60, start_phone_stats_home) if hs.config.daemonize and hs.config.print_pidfile: print(hs.config.pid_file) _base.start_reactor( "synapse-homeserver", hs.config.soft_file_limit, hs.config.gc_thresholds, hs.config.pid_file, hs.config.daemonize, hs.config.cpu_affinity, logger, )
def run(hs): PROFILE_SYNAPSE = False if PROFILE_SYNAPSE: def profile(func): from cProfile import Profile from threading import current_thread def profiled(*args, **kargs): profile = Profile() profile.enable() func(*args, **kargs) profile.disable() ident = current_thread().ident profile.dump_stats("/tmp/%s.%s.%i.pstat" % ( hs.hostname, func.__name__, ident )) return profiled from twisted.python.threadpool import ThreadPool ThreadPool._worker = profile(ThreadPool._worker) reactor.run = profile(reactor.run) clock = hs.get_clock() start_time = clock.time() stats = {} # Contains the list of processes we will be monitoring # currently either 0 or 1 stats_process = [] @defer.inlineCallbacks def phone_stats_home(): logger.info("Gathering stats for reporting") now = int(hs.get_clock().time()) uptime = int(now - start_time) if uptime < 0: uptime = 0 stats["homeserver"] = hs.config.server_name stats["timestamp"] = now stats["uptime_seconds"] = uptime stats["total_users"] = yield hs.get_datastore().count_all_users() total_nonbridged_users = yield hs.get_datastore().count_nonbridged_users() stats["total_nonbridged_users"] = total_nonbridged_users room_count = yield hs.get_datastore().get_room_count() stats["total_room_count"] = room_count stats["daily_active_users"] = yield hs.get_datastore().count_daily_users() stats["daily_active_rooms"] = yield hs.get_datastore().count_daily_active_rooms() stats["daily_messages"] = yield hs.get_datastore().count_daily_messages() r30_results = yield hs.get_datastore().count_r30_users() for name, count in r30_results.iteritems(): stats["r30_users_" + name] = count daily_sent_messages = yield hs.get_datastore().count_daily_sent_messages() stats["daily_sent_messages"] = daily_sent_messages stats["cache_factor"] = CACHE_SIZE_FACTOR stats["event_cache_size"] = hs.config.event_cache_size if len(stats_process) > 0: stats["memory_rss"] = 0 stats["cpu_average"] = 0 for process in stats_process: stats["memory_rss"] += process.memory_info().rss stats["cpu_average"] += int(process.cpu_percent(interval=None)) logger.info("Reporting stats to matrix.org: %s" % (stats,)) try: yield hs.get_simple_http_client().put_json( "https://matrix.org/report-usage-stats/push", stats ) except Exception as e: logger.warn("Error reporting stats: %s", e) def performance_stats_init(): try: import psutil process = psutil.Process() # Ensure we can fetch both, and make the initial request for cpu_percent # so the next request will use this as the initial point. process.memory_info().rss process.cpu_percent(interval=None) logger.info("report_stats can use psutil") stats_process.append(process) except (ImportError, AttributeError): logger.warn( "report_stats enabled but psutil is not installed or incorrect version." " Disabling reporting of memory/cpu stats." " Ensuring psutil is available will help matrix.org track performance" " changes across releases." ) if hs.config.report_stats: logger.info("Scheduling stats reporting for 3 hour intervals") clock.looping_call(phone_stats_home, 3 * 60 * 60 * 1000) # We need to defer this init for the cases that we daemonize # otherwise the process ID we get is that of the non-daemon process clock.call_later(0, performance_stats_init) # We wait 5 minutes to send the first set of stats as the server can # be quite busy the first few minutes clock.call_later(5 * 60, phone_stats_home) if hs.config.daemonize and hs.config.print_pidfile: print (hs.config.pid_file) _base.start_reactor( "synapse-homeserver", hs.config.soft_file_limit, hs.config.gc_thresholds, hs.config.pid_file, hs.config.daemonize, hs.config.cpu_affinity, logger, )