def update_process_list():
global PS
plist = []
for pid in psutil.get_pid_list():
try:
p = Process(pid)
plist.append((p,p.get_cpu_percent(),p.get_memory_percent()))
except psutil.NoSuchProcess:
continue
# copy plist to PS
PS = sorted(plist[:], key=lambda e: e[1] if e[1] > e[2] else e[2], reverse=True)[:25]
def update_process_list():
global PS
plist = []
for pid in psutil.get_pid_list():
try:
p = Process(pid)
plist.append((p, p.get_cpu_percent(), p.get_memory_percent()))
except psutil.NoSuchProcess:
continue
# copy plist to PS
PS = sorted(plist[:],
key=lambda e: e[1] if e[1] > e[2] else e[2],
reverse=True)[:25]
def _on_server(self, args):
os.chdir(self.original_dir)
abs_args_path = os.path.abspath(args.path)
component_details = open(os.path.join(abs_args_path, ZATO_INFO_FILE)).read()
out = {
'component_details': component_details,
'component_full_path': abs_args_path,
'component_host': current_host(),
'component_running': False,
'current_time': datetime.now().isoformat(),
'current_time_utc': datetime.utcnow().isoformat(),
'master_proc_connections': None,
'master_proc_pid': None,
'master_proc_name': None,
'master_proc_create_time': None,
'master_proc_create_time_utc': None,
'master_proc_username': None,
'master_proc_workers_no': None,
'master_proc_workers_pids': None,
}
master_proc_pid = self._zdaemon_command('status')
master_proc_pid = master_proc_pid.values()
if master_proc_pid and master_proc_pid[0]:
out['component_running'] = True
master_proc_pid = int(master_proc_pid[0])
master_proc = Process(master_proc_pid)
workers_pids = sorted(elem.pid for elem in master_proc.get_children())
out['master_proc_connections'] = master_proc.get_connections()
out['master_proc_pid'] = master_proc.pid
out['master_proc_create_time'] = datetime.fromtimestamp(master_proc.create_time).isoformat()
out['master_proc_create_time_utc'] = datetime.fromtimestamp(master_proc.create_time, UTC).isoformat()
out['master_proc_username'] = master_proc.username
out['master_proc_name'] = master_proc.name
out['master_proc_workers_no'] = len(workers_pids)
out['master_proc_workers_pids'] = workers_pids
for pid in workers_pids:
worker = Process(pid)
worker_memory_percent = worker.get_memory_percent()
out['worker_{}_create_time'.format(pid)] = datetime.fromtimestamp(worker.create_time).isoformat()
out['worker_{}_create_time_utc'.format(pid)] = datetime.fromtimestamp(worker.create_time, UTC).isoformat()
out['worker_{}_connections'.format(pid)] = worker.get_connections()
if getattr(args, 'json', False):
out['component_details'] = loads(out['component_details'])
self.logger.info(dumps(out))
else:
cols_width = args.cols_width if args.cols_width else DEFAULT_COLS_WIDTH
cols_width = (elem.strip() for elem in cols_width.split(','))
cols_width = [int(elem) for elem in cols_width]
table = Texttable()
table.set_cols_width(cols_width)
# Use text ('t') instead of auto so that boolean values don't get converted into ints
table.set_cols_dtype(['t', 't'])
rows = [['Key', 'Value']]
rows.extend(sorted(out.items()))
table.add_rows(rows)
self.logger.info(table.draw())
def get_info(process=None, interval=0):
"""Return information about a process.
If process is None, will return the information about the current process
"""
if process is None:
process = Process(os.getpid())
info = {}
try:
mem_info = process.get_memory_info()
info['mem_info1'] = bytes2human(mem_info[0])
info['mem_info2'] = bytes2human(mem_info[1])
except AccessDenied:
info['mem_info1'] = info['mem_info2'] = "N/A"
try:
info['cpu'] = process.get_cpu_percent(interval=interval)
except AccessDenied:
info['cpu'] = "N/A"
try:
info['mem'] = round(process.get_memory_percent(), 1)
except AccessDenied:
info['mem'] = "N/A"
try:
cpu_times = process.get_cpu_times()
ctime = timedelta(seconds=sum(cpu_times))
ctime = "%s:%s.%s" % (ctime.seconds // 60 % 60,
str((ctime.seconds % 60)).zfill(2),
str(ctime.microseconds)[:2])
except AccessDenied:
ctime = "N/A"
info['ctime'] = ctime
try:
info['pid'] = process.pid
except AccessDenied:
info['pid'] = 'N/A'
try:
info['username'] = process.username
except AccessDenied:
info['username'] = '******'
try:
info['nice'] = process.nice
except AccessDenied:
info['nice'] = 'N/A'
except NoSuchProcess:
info['nice'] = 'Zombie'
try:
cmdline = os.path.basename(shlex.split(process.cmdline[0])[0])
except (AccessDenied, IndexError):
cmdline = "N/A"
info['cmdline'] = cmdline
info['children'] = []
for child in process.get_children():
info['children'].append(get_info(child, interval=interval))
return info
def _on_server(self, args):
os.chdir(self.original_dir)
abs_args_path = os.path.abspath(args.path)
component_details = open(os.path.join(abs_args_path,
ZATO_INFO_FILE)).read()
out = {
'component_details': component_details,
'component_full_path': abs_args_path,
'component_host': current_host(),
'component_running': False,
'current_time': datetime.now().isoformat(),
'current_time_utc': datetime.utcnow().isoformat(),
'master_proc_connections': None,
'master_proc_pid': None,
'master_proc_name': None,
'master_proc_create_time': None,
'master_proc_create_time_utc': None,
'master_proc_username': None,
'master_proc_workers_no': None,
'master_proc_workers_pids': None,
}
master_proc_pid = self._zdaemon_command('status')
master_proc_pid = master_proc_pid.values()
if master_proc_pid and master_proc_pid[0]:
out['component_running'] = True
master_proc_pid = int(master_proc_pid[0])
master_proc = Process(master_proc_pid)
workers_pids = sorted(elem.pid
for elem in master_proc.get_children())
out['master_proc_connections'] = master_proc.get_connections()
out['master_proc_pid'] = master_proc.pid
out['master_proc_create_time'] = datetime.fromtimestamp(
master_proc.create_time).isoformat()
out['master_proc_create_time_utc'] = datetime.fromtimestamp(
master_proc.create_time, UTC).isoformat()
out['master_proc_username'] = master_proc.username
out['master_proc_name'] = master_proc.name
out['master_proc_workers_no'] = len(workers_pids)
out['master_proc_workers_pids'] = workers_pids
for pid in workers_pids:
worker = Process(pid)
worker_memory_percent = worker.get_memory_percent()
out['worker_{}_create_time'.format(
pid)] = datetime.fromtimestamp(
worker.create_time).isoformat()
out['worker_{}_create_time_utc'.format(
pid)] = datetime.fromtimestamp(worker.create_time,
UTC).isoformat()
out['worker_{}_connections'.format(
pid)] = worker.get_connections()
if getattr(args, 'json', False):
out['component_details'] = loads(out['component_details'])
self.logger.info(dumps(out))
else:
cols_width = args.cols_width if args.cols_width else DEFAULT_COLS_WIDTH
cols_width = (elem.strip() for elem in cols_width.split(','))
cols_width = [int(elem) for elem in cols_width]
table = Texttable()
table.set_cols_width(cols_width)
# Use text ('t') instead of auto so that boolean values don't get converted into ints
table.set_cols_dtype(['t', 't'])
rows = [['Key', 'Value']]
rows.extend(sorted(out.items()))
table.add_rows(rows)
self.logger.info(table.draw())
class Memory:
"""
Cache内存管理类
"""
def __init__(self, dbname):
"""
构造函数
@param dbname: 数据库地址
"""
self.dbname = dbname # 当前DB名称
self.db = leveldb.LevelDB(dbname)
self.savetime = 0
self.caches = {} # 缓存数据Hash
self.keys = set() # 修改过的Key, 需要Dump数据
self.delkeys = set() # 删除过的Key, 需要删除数据
self.expirekeys = set() # 可以过期的Key
self.start_time = time.time()
self.version = None
self.last_cmd_time = 0
self.process = Process(os.getpid())
self.load_db()
# ==============DB================
@timeit
def load_db(self):
"""
从数据库载入数据
"""
print self.db.GetStats()
for key, val in self.db.RangeIter():
val = self.unserialize(val)
#print val
if val[FIELD_EXPIRE] == 0 or time.time() < val[FIELD_EXPIRE]:
self.caches[key] = val
@timeit
def dump_db(self):
"""
写入K/V数据到数据库
"""
keys, delkeys = self.key_process()
batch = leveldb.WriteBatch()
# 优化已过期的Key不再写入DB
for key, val in self.filter_keys(keys):
batch.Put(key, val)
map(batch.Delete, set(delkeys))
self.db.Write(batch, sync=False)
self.savetime = time.time()
#print self.caches
def filter_keys(self, keys):
"""
过滤失效的Key并序列化数据
"""
for key in set(keys):
if key in self.caches and \
(self.caches[key][FIELD_EXPIRE] == 0 or time.time() < self.caches[key][FIELD_EXPIRE]):
yield (key, self.serialize(self.caches[key]))
@staticmethod
def unserialize(val):
"""
反序列化数据
"""
data = msgpack.unpackb(val)
return data
@staticmethod
def serialize(val):
"""
序列化数据
"""
#print 's=>%s' % key
data = msgpack.packb(val)
return data
@lock
def key_process(self):
"""
Key处理, 准备Dump
"""
keys = copy.deepcopy(self.keys)
self.keys = set()
delkeys = copy.deepcopy(self.delkeys)
self.delkeys = set()
return keys, delkeys
# ==============String================
def set(self, key, val, expire, flag):
self.keys.add(key)
#print val
if expire == 0:
self.caches[key] = [val, expire, flag, SRTING_TYPE]
else:
self.caches[key] = [val, 0, flag, SRTING_TYPE]
self.expire(key, expire)
def get(self, key):
if key in self.caches:
if self.check_expire(key):
return -1, None
data = self.caches[key][FIELD_VALUE]
if data is None:
self.delete(key) # 已过期
return 0, None
else:
return 1, data
else:
return 0, None
def delete(self, key):
self.delkeys.add(key)
if key in self.caches:
del self.caches[key]
return 1
else:
return 0
def exists(self, key):
if key in self.caches:
return 1
else:
return 0
def expire(self, key, expire):
try:
etime = int(time.time()) + expire
self.caches[key][FIELD_EXPIRE] = etime # 过期时间
self.expirekeys.add(key) # 可以过期的Key标记一下
return 1, etime
except KeyError:
return 0, None
def persist(self, key):
try:
self.caches[key][FIELD_EXPIRE] = 0 # 不过期
return 1
except KeyError:
return 0
def ttl(self, key):
try:
return 1, self.caches[key][FIELD_EXPIRE]
except KeyError:
return 0, None
def rename(self, key, newkey):
if key in self.caches:
if newkey not in self.caches:
self.caches[newkey] = self.caches[key]
self.delete(key)
return 1
else:
return 2
else:
return 0
def check_expire(self, key):
code, expire = self.ttl(key)
if code == 1:
if expire != 0 and time.time() > expire: # 过期
self.delete(key) # 过期的Key删除
self.expirekeys.remove(key) # 可以过期的key列表删除
return True
else:
return False
else:
return False
# ==============List================
def lpush(self, key, val):
if key in self.caches:
self.keys.add(key)
if self.caches[key][FIELD_TYPE] == LIST_TYPE:
self.caches[key][FIELD_VALUE].append(val)
return 1
else:
return 0
else:
self.keys.add(key)
self.caches[key] = [[val], 0, 0, LIST_TYPE]
return 1
def lpop(self, key):
if key in self.caches:
self.keys.add(key)
if self.caches[key][FIELD_TYPE] == LIST_TYPE:
try:
val = self.caches[key][FIELD_VALUE].pop()
return 1, val
except IndexError:
return -1, None
else:
return 0, None
else:
return 0, None
def lrange(self, key, start, stop):
if key in self.caches:
if self.caches[key][FIELD_TYPE] == LIST_TYPE:
if stop == -1:
return 1, self.caches[key][FIELD_VALUE][start:]
else:
return 1, self.caches[key][FIELD_VALUE][start:stop]
else:
return 2, None
else:
return 0, None
def llen(self, key):
if key in self.caches:
if self.caches[key][FIELD_TYPE] == LIST_TYPE:
return 1, len(self.caches[key][FIELD_VALUE])
else:
return 2, None
else:
return 0, None
def lindex(self, key, index):
if key in self.caches:
if self.caches[key][FIELD_TYPE] == LIST_TYPE:
try:
return 1, self.caches[key][FIELD_VALUE][index]
except IndexError:
return 3, None
else:
return 2, None
else:
return 0, None
def linsert(self, key, index, val):
if key in self.caches:
self.keys.add(key)
if self.caches[key][FIELD_TYPE] == LIST_TYPE:
self.caches[key][FIELD_VALUE].insert(index, val)
return 1, None
else:
return 2, None
else:
return 0, None
# ==============Hash================
def hmset(self, key, values):
self.keys.add(key)
if key in self.caches:
if self.caches[key][FIELD_TYPE] == HASH_TYPE:
self.caches[key][FIELD_VALUE].update(values)
return 1, None
else:
return 0, None
else:
self.caches[key] = [values, 0, 0, HASH_TYPE]
return 1, None
def hset(self, key, field, val):
if key in self.caches:
self.keys.add(key)
if self.caches[key][FIELD_TYPE] == HASH_TYPE:
self.caches[key][FIELD_VALUE][field] = val
return 1, None
else:
return 2, None
else:
self.keys.add(key)
self.caches[key] = [{field: val}, 0, 0, HASH_TYPE]
return 1, None
def hget(self, key, fields):
if key in self.caches:
if self.caches[key][FIELD_TYPE] == HASH_TYPE:
data = {field: self.caches[key][FIELD_VALUE][field] for field in fields if field in self.caches[key][FIELD_VALUE]}
return 1, data
else:
return 0, None
else:
return 0, None
def hgetall(self, key):
if key in self.caches:
if self.caches[key][FIELD_TYPE] == HASH_TYPE:
return 1, self.caches[key][FIELD_VALUE]
else:
return 0, None
else:
return 0, None
def hexists(self, key, field):
if key in self.caches:
if self.caches[key][FIELD_TYPE] == HASH_TYPE:
#print self.caches[key][FIELD_VALUE], field
if field in self.caches[key][FIELD_VALUE]:
return 1, 1
else:
return 1, 0
else:
return 0, None
else:
return 0, None
def hlen(self, key):
if key in self.caches:
if self.caches[key][FIELD_TYPE] == HASH_TYPE:
return 1, len(self.caches[key][FIELD_VALUE])
else:
return 0, None
else:
return 0, None
def hdel(self, key, fields):
if key in self.caches:
if self.caches[key][FIELD_TYPE] == HASH_TYPE:
for field in fields:
try:
del self.caches[key][FIELD_VALUE][field]
except KeyError:
pass
return 1, None
else:
return 0, None
else:
return 0, None
def hkeys(self, key):
if key in self.caches:
if self.caches[key][FIELD_TYPE] == HASH_TYPE:
return 1, self.caches[key][FIELD_VALUE].keys()
else:
return 0, None
else:
return 0, None
def hvals(self, key):
if key in self.caches:
if self.caches[key][FIELD_TYPE] == HASH_TYPE:
return 1, self.caches[key][FIELD_VALUE].values()
else:
return 0, None
else:
return 0, None
def get_status(self):
from pycached import __version__
status = {
'last_cmd_time': self.last_cmd_time,
'cmd_version': self.version,
'uptime_in_seconds': int(time.time() - self.start_time),
'process_id': self.process.pid,
'pycached_version': __version__,
'key_size': len(self.caches),
'next_save_key_size': len(self.keys),
'next_del_key_size': len(self.delkeys),
'expire_key_size': len(self.expirekeys),
'save_time': self.savetime,
'threads': self.process.get_num_threads(),
'used_memory': self.process.get_ext_memory_info().rss,
'used_memory_peak':self.process.get_ext_memory_info().vms,
'memory_percent': "%0.4f" % self.process.get_memory_percent(),
#'used_cpu': cpu_percent(),
}
return status
def change_version(self):
self.last_cmd_time = int(time.time())
self.version = ''.join(random.sample(string.ascii_letters, 16))
