def collect_statistics_for_db(cls, debug=False): lag = db_functions.mongo_max_replication_lag(settings.MONGODB) cls.set('mongodb_replication_lag', lag) now = round_time(datetime.datetime.now(), round_to=60) r = redis.Redis(connection_pool=settings.REDIS_STATISTICS_POOL) db_times = {} latest_db_times = {} for db in [ 'sql', 'mongo', 'redis', 'task_sql', 'task_mongo', 'task_redis' ]: db_times[db] = [] for hour in range(24): start_hours_ago = now - datetime.timedelta(hours=hour + 1) pipe = r.pipeline() for m in range(60): minute = start_hours_ago + datetime.timedelta(minutes=m) key = "DB:%s:%s" % (db, minute.strftime('%s')) if debug: print(" -> %s:c" % key) pipe.get("%s:c" % key) pipe.get("%s:t" % key) times = pipe.execute() counts = [int(c or 0) for c in times[::2]] avgs = [float(a or 0) for a in times[1::2]] if counts and avgs: count = sum(counts) avg = round(sum(avgs) / count, 3) if count else 0 else: count = 0 avg = 0 if hour == 0: latest_count = float(counts[-1]) if len(counts) else 0 latest_avg = float(avgs[-1]) if len(avgs) else 0 latest_db_times[ db] = latest_avg / latest_count if latest_count else 0 db_times[db].append(avg) db_times[db].reverse() values = ( ('avg_sql_times', json.encode(db_times['sql'])), ('avg_mongo_times', json.encode(db_times['mongo'])), ('avg_redis_times', json.encode(db_times['redis'])), ('latest_sql_avg', latest_db_times['sql']), ('latest_mongo_avg', latest_db_times['mongo']), ('latest_redis_avg', latest_db_times['redis']), ('latest_task_sql_avg', latest_db_times['task_sql']), ('latest_task_mongo_avg', latest_db_times['task_mongo']), ('latest_task_redis_avg', latest_db_times['task_redis']), ) for key, value in values: cls.objects(key=key).update_one(upsert=True, set__key=key, set__value=value)
def collect_statistics_sites_loaded(cls): now = round_time(datetime.datetime.now(), round_to=60) sites_loaded = [] avg_time_taken = [] last_5_min_time_taken = 0 r = redis.Redis(connection_pool=settings.REDIS_STATISTICS_POOL) for hour in range(24): start_hours_ago = now - datetime.timedelta(hours=hour + 1) pipe = r.pipeline() for m in range(60): minute = start_hours_ago + datetime.timedelta(minutes=m) key = "%s:%s" % (RStats.stats_type('page_load'), minute.strftime('%s')) pipe.get("%s:s" % key) pipe.get("%s:a" % key) times = pipe.execute() counts = [int(c) for c in times[::2] if c] avgs = [float(a) for a in times[1::2] if a] if hour == 0: last_5_min_time_taken = round( sum(avgs[:1]) / max(1, sum(counts[:1])), 2) if counts and avgs: count = max(1, sum(counts)) avg = round(sum(avgs) / count, 3) else: count = 0 avg = 0 sites_loaded.append(count) avg_time_taken.append(avg) sites_loaded.reverse() avg_time_taken.reverse() values = ( ('sites_loaded', json.encode(sites_loaded)), ('avg_time_taken', json.encode(avg_time_taken)), ('latest_sites_loaded', sites_loaded[-1]), ('latest_avg_time_taken', avg_time_taken[-1]), ('max_sites_loaded', max(sites_loaded)), ('max_avg_time_taken', max(1, max(avg_time_taken))), ('last_5_min_time_taken', last_5_min_time_taken), ) for key, value in values: cls.objects(key=key).update_one(upsert=True, set__key=key, set__value=value)
def collect_statistics_for_db(cls): lag = db_functions.mongo_max_replication_lag(settings.MONGODB) cls.set('mongodb_replication_lag', lag) now = round_time(datetime.datetime.now(), round_to=60) r = redis.Redis(connection_pool=settings.REDIS_STATISTICS_POOL) db_times = {} latest_db_times = {} for db in ['sql', 'mongo', 'redis', 'task_sql', 'task_mongo', 'task_redis']: db_times[db] = [] for hour in range(24): start_hours_ago = now - datetime.timedelta(hours=hour+1) pipe = r.pipeline() for m in range(60): minute = start_hours_ago + datetime.timedelta(minutes=m) key = "DB:%s:%s" % (db, minute.strftime('%s')) pipe.get("%s:c" % key) pipe.get("%s:t" % key) times = pipe.execute() counts = [int(c or 0) for c in times[::2]] avgs = [float(a or 0) for a in times[1::2]] if counts and avgs: count = sum(counts) avg = round(sum(avgs) / count, 3) if count else 0 else: count = 0 avg = 0 if hour == 0: latest_count = float(counts[-1]) if len(counts) else 0 latest_avg = float(avgs[-1]) if len(avgs) else 0 latest_db_times[db] = latest_avg / latest_count if latest_count else 0 db_times[db].append(avg) db_times[db].reverse() values = ( ('avg_sql_times', json.encode(db_times['sql'])), ('avg_mongo_times', json.encode(db_times['mongo'])), ('avg_redis_times', json.encode(db_times['redis'])), ('latest_sql_avg', latest_db_times['sql']), ('latest_mongo_avg', latest_db_times['mongo']), ('latest_redis_avg', latest_db_times['redis']), ('latest_task_sql_avg', latest_db_times['task_sql']), ('latest_task_mongo_avg', latest_db_times['task_mongo']), ('latest_task_redis_avg', latest_db_times['task_redis']), ) for key, value in values: cls.objects(key=key).update_one(upsert=True, set__key=key, set__value=value)
def _save_times(self, db_times): if not db_times: return r = redis.Redis(connection_pool=settings.REDIS_STATISTICS_POOL) pipe = r.pipeline() minute = round_time(round_to=60) for db, duration in db_times.items(): key = "DB:%s:%s" % (db, minute.strftime('%s')) pipe.incr("%s:c" % key) pipe.expireat("%s:c" % key, (minute + datetime.timedelta(days=2)).strftime("%s")) if duration: pipe.incrbyfloat("%s:t" % key, duration) pipe.expireat("%s:t" % key, (minute + datetime.timedelta(days=2)).strftime("%s")) pipe.execute()
def collect_statistics_sites_loaded(cls): now = round_time(datetime.datetime.now(), round_to=60) sites_loaded = [] avg_time_taken = [] last_5_min_time_taken = 0 r = redis.Redis(connection_pool=settings.REDIS_STATISTICS_POOL) for hour in range(24): start_hours_ago = now - datetime.timedelta(hours=hour+1) pipe = r.pipeline() for m in range(60): minute = start_hours_ago + datetime.timedelta(minutes=m) key = "%s:%s" % (RStats.stats_type('page_load'), minute.strftime('%s')) pipe.get("%s:s" % key) pipe.get("%s:a" % key) times = pipe.execute() counts = [int(c) for c in times[::2] if c] avgs = [float(a) for a in times[1::2] if a] if hour == 0: last_5_min_time_taken = round(sum(avgs[:1]) / max(1, sum(counts[:1])), 2) if counts and avgs: count = max(1, sum(counts)) avg = round(sum(avgs) / count, 3) else: count = 0 avg = 0 sites_loaded.append(count) avg_time_taken.append(avg) sites_loaded.reverse() avg_time_taken.reverse() values = ( ('sites_loaded', json.encode(sites_loaded)), ('avg_time_taken', json.encode(avg_time_taken)), ('latest_sites_loaded', sites_loaded[-1]), ('latest_avg_time_taken', avg_time_taken[-1]), ('max_sites_loaded', max(sites_loaded)), ('max_avg_time_taken', max(1, max(avg_time_taken))), ('last_5_min_time_taken', last_5_min_time_taken), ) for key, value in values: cls.objects(key=key).update_one(upsert=True, set__key=key, set__value=value)