def __init__(self,
db_host='127.0.0.1',
log_file="/tmp/api-server.log",
log_level=logging.NOTSET):
# cassandra database object
self.db = None
self.db_host = db_host
self.buf = HallBuffer()
def __init__(self,
mysql_cfg={},
log_file="/tmp/api-server.log",
log_level=logging.NOTSET):
# cassandra database object
self.db = None
self.mysql_cfg=mysql_cfg;
self.buf = HallBuffer()
self.logger = logging.getLogger()
handler = logging.FileHandler(log_file)
self.logger.addHandler(handler)
self.logger.setLevel(log_level)
def __init__(self,
db_host='127.0.0.1',
log_file="/tmp/api-server.log",
log_level=logging.NOTSET):
# cassandra database object
self.db = None
self.db_host = db_host
self.buf = HallBuffer()
self.logger = logging.getLogger()
handler = logging.FileHandler(log_file)
self.logger.addHandler(handler)
self.logger.setLevel(log_level)
class ApiServer():
def __init__(self,
mysql_cfg={},
log_file="/tmp/api-server.log",
log_level=logging.NOTSET):
# cassandra database object
self.db = None
self.mysql_cfg=mysql_cfg;
self.buf = HallBuffer()
self.logger = logging.getLogger()
handler = logging.FileHandler(log_file)
self.logger.addHandler(handler)
self.logger.setLevel(log_level)
def get_db(self):
if self.db is None:
self.db=MysqlDb(self.mysql_cfg);
# self.db = CassaDb('data', self.db_host)
return self.db
### change by lanjinsong 2012-08-10
def get_instances_list(self,start_time=time.strftime("%Y-%m-%d %T",time.gmtime(0))):
""" show all instances in the databases"""
db=self.get_db();
return self.db.get_instances_in_cache();
#result_set=db.get_all_instances(start_time);
#print type(result_set),'result_set=',result_set;
#instances=[];
#for row in result_set:
# instances.append(row[0]);
#return instances;
def __get_instances_list(self, cf_str):
if not isinstance(cf_str, unicode):
print 'param types error'
return None
ret = list()
limit = 20000
time_to = int(time.time())
time_from = time_to - 24 * 60 * 60
db = self.get_db()
rs = db.get_range2(cf_str, row_count=20)
return list(rs)
if not rs is None:
print rs
for i in rs:
ret.append(i[0])
return ret
def __get_instances_list(self, cf_str):
if not isinstance(cf_str, unicode):
print 'param types error'
return None
ret = list()
limit = 20000
time_to = int(time.time())
time_from = time_to - 24 * 60 * 60
db = self.get_db()
rs = db.get_range2(cf_str, row_count=20)
return list(rs)
if not rs is None:
print rs
for i in rs:
ret.append(i[0])
return ret
def get_by_instance_id(self, row_id, cf_str):
if not isinstance(row_id, unicode) \
or not isinstance(cf_str, unicode):
print 'param types error'
return None, 0, True
db = self.get_db()
rs = db.getbykey(cf_str, row_id)
count = 0 if rs is None else len(rs)
return rs, count, False if (count == 20000) else True
## change by lanjinsong
def query_usage_report(self,args,**kwargs):
""" query usage report modified by lanjinsong to use MySQL"""
instance_id=str(args['instance_id']);
start_time=args['start_time'] if ('start_time' in args) else time.strftime("%Y-%m-%d %T",time.gmtime(0));
db=self.get_db();
#result_set=db.get_instance_info(instance_id,start_time);
result_set=db.get_instance_info_in_cache(instance_id);
# print 'instance_id=%s start_time=%s'%(instance_id,start_time);
# print 'result_set=',result_set;
return result_set;
### TODO: decode it
results=[];
for row in result_set:
info=list(row);
# print 'info=',info,'type of info[-1]=',type(info[-1]);
info[-1]=info[-1].strftime("%Y-%m-%d %T");
results.append(info);
return results;
def __query_usage_report(self, args, **kwargs):
# TODO: how to use kwargs?
# def query_usage_report(self, arg, id=None, metric='cpu',
# metric_param='total',
# statistic='avg', period=5,
# timestamp_from=None, timestamp_to=None,
# **kwargs):
"""statistic is STATISTIC enum
period default=5 minutes
time_to default=0(now)"""
"""
{
'id': 'instance00001',
'metric': 'network',
'metric_param': 'vnet0',
'statistic': 'sum',
'period': 5,
'timestamp_from': '2012-02-20T12:12:12',
'timestamp_to': '2012-02-22T12:12:12',
}
"""
# usage_report = dict()
# datetime_from = iso8601.parse_date(timestamp_from)
# datetime_to = iso8601.parse_date(timestamp_to)
# # TODO: implement
# return {'data': usage_report}
row_id = args['id']
cf_str = args['metric']
scf_str = args['metric_param']
statistic = args['statistic']
period = int(args['period'])
timestamp_from = args['timestamp_from']
timestamp_to = args['timestamp_to']
time_from = iso8601.parse_date(timestamp_from)
time_from = int(time.mktime(time_from.timetuple()))
time_to = int(time.time())
if not timestamp_to is None:
time_to = iso8601.parse_date(timestamp_to)
time_to = int(time.mktime(time_to.timetuple()))
bufkey = str([row_id, cf_str, scf_str,
statistic, period, time_from, time_to])
if self.buf.hit_test(bufkey):
print "buffer hit:", bufkey
return self.buf.get_buf(bufkey)
ret_len = 0
(rs, count, all_data) = self.get_data(row_id, cf_str, scf_str,
time_from, time_to)
if not rs is None and count > 0:
buf = self.analyize_data(rs, 1, statistic)
ret = self.analyize_data(buf, period, statistic)
if ret is None:
ret_len = 0
else:
ret = OrderedDict(sorted(ret.items(), key=lambda t: t[0]))
ret_len = len(ret)
print ret_len, "result."
else:
print "no result."
ret = None
ret_len = 0
result = ret, ret_len, all_data
if (not result is None and time.time() - time_to > 120):
self.buf.cleanup()
self.buf.save(bufkey, result)
return result
def testHallBuffer(self):
params = ["iphone", 2012]
data = "detail of 2012 iphone 4S"
b = HallBuffer()
key = str(params)
print "Hit rate=", b.get_hit_rate(params), b.get_hit_rate(key), data
self.assertEqual(b.get_hit_rate(params), 0)
if b.hit_test(key):
print b.get_buf(key);
else:
b.save(key, data)
print "not hit."
print "Hit rate=", b.get_hit_rate(params), b.get_hit_rate(key), data
if b.hit_test(key):
print "hit!"
print b.get_buf(key);
else:
b.save(key, data)
print "Hit rate=", b.get_hit_rate(params), b.get_hit_rate(key), data
self.assertEqual(b.get_hit_rate(params), 1)
b.cleanup(time_out = 0)
print "after cleanup, Hit rate=", b.get_hit_rate(params), data, b.get_hit_rate(key)
self.assertEqual(b.get_hit_rate(params), 0)
def testHallBuffer(self):
params = ["iphone", 2012]
data = "detail of 2012 iphone 4S"
b = HallBuffer()
key = str(params)
print "Hit rate=", b.get_hit_rate(params), b.get_hit_rate(key), data
self.assertEqual(b.get_hit_rate(params), 0)
if b.hit_test(key):
print b.get_buf(key)
else:
b.save(key, data)
print "not hit."
print "Hit rate=", b.get_hit_rate(params), b.get_hit_rate(key), data
if b.hit_test(key):
print "hit!"
print b.get_buf(key)
else:
b.save(key, data)
print "Hit rate=", b.get_hit_rate(params), b.get_hit_rate(key), data
self.assertEqual(b.get_hit_rate(params), 1)
b.cleanup(time_out=0)
print "after cleanup, Hit rate=", b.get_hit_rate(
params), data, b.get_hit_rate(key)
self.assertEqual(b.get_hit_rate(params), 0)
class ApiServer():
def __init__(self,
db_host='127.0.0.1',
log_file="/tmp/api-server.log",
log_level=logging.NOTSET):
# cassandra database object
self.db = None
self.db_host = db_host
self.buf = HallBuffer()
def get_db(self):
if self.db is None:
self.db = CassaDb('data', self.db_host)
return self.db
def get_data(self, row_id, cf_str, scf_str, time_from=0, time_to=0):
"""
param type: UnicodeType and IntType
return: recordset, count, bool(count > limit?)
"""
if not isinstance(row_id, unicode) \
or not isinstance(cf_str, unicode) \
or not isinstance(scf_str, unicode) \
or not isinstance(time_from, int) \
or not isinstance(time_to, int):
return None, 0, True
if time_to == 0:
time_to = int(utils.utcnow_ts())
print "get_data:", row_id, cf_str, scf_str, time_from, time_to
bufkey = str([row_id, cf_str, scf_str, time_from, time_to])
if self.buf.hit_test(bufkey):
return self.buf.get_buf(bufkey)
db = self.get_db()
rs = db.get(cf_str, row_id, super_column=scf_str,
column_start=time_from, column_finish=time_to, column_count=20000)
count = 0 if rs is None else len(rs)
ret = rs, count, False if (count == 20000) else True
if not rs is None:
self.buf.save(bufkey, ret)
return ret
def analyize_data(self, rs, period, statistic, show=False):
"""[private func]analyize the data
period: minutes
return: {"key":"value"}
note: timestmp is already UTC time, so do NOT use time.gmtime again!
"""
if rs is None \
or not isinstance(period, int):
return None
t = 0
key_time = 0
st = Statistics()
this_period = dict()
for timestmp, value in rs.iteritems():
rt = time.localtime(timestmp)
key = rt.tm_min + rt.tm_hour*100 + rt.tm_mday*10000 + \
rt.tm_mon*1000000 + rt.tm_year*100000000
if t == 0:
st.clean()
t = timestmp
key_time = time.localtime(timestmp)
if timestmp >= t + period*60:
st.clean()
t = timestmp
key_time = time.localtime(timestmp)
st.update(float(value))
key2 = time.mktime(
(key_time.tm_year, key_time.tm_mon, key_time.tm_mday,
key_time.tm_hour, key_time.tm_min,0,0,0,0))
this_period[key2] = st.get_value(statistic)
this_period = OrderedDict(sorted(this_period.items(), key=lambda t: t[0]))
if show:
print statistic, ":each period(", period, "):"
for m, val in this_period.iteritems():
rt = time.localtime(m)
key = rt.tm_min + rt.tm_hour*100 + rt.tm_mday*10000 + \
rt.tm_mon*1000000 + rt.tm_year*100000000
if show:
print '\t', key, m, val
return this_period
######################### public API interface ########################
def get_instances_list(self, cf_str):
if not isinstance(cf_str, unicode):
print 'param types error'
return None
ret = list()
limit = 20000
time_to = int(utils.utcnow_ts())
time_from = time_to - 24 * 60 * 60
db = self.get_db()
rs = db.get_range2(cf_str, row_count=20)
return list(rs)
if not rs is None:
for i in rs:
ret.append(i[0])
return ret
def get_by_instance_id(self, row_id, cf_str):
if not isinstance(row_id, unicode) \
or not isinstance(cf_str, unicode):
print 'param types error'
return None, 0, True
db = self.get_db()
rs = db.getbykey(cf_str, row_id)
count = 0 if rs is None else len(rs)
return rs, count, False if (count == 20000) else True
def get_by_key(self, row_id, cf_str, scf_str, limit=20000):
"""
example:cf=u'vmnetwork',scf=u'10.0.0.1',key=u'instance-0000002'
return: recordset, count, bool(count > limit?)
"""
if not isinstance(row_id, unicode) \
or not isinstance(cf_str, unicode) \
or not isinstance(scf_str, unicode) \
or not isinstance(limit, int):
print 'param types error'
return None, 0, True
db = self.get_db()
rs = db.getbykey2(cf_str, key=row_id, super_column=scf_str, column_count=limit)
count = 0 if rs is None else len(rs)
return rs, count, False if (count == 20000) else True
def query_usage_report(self, args, **kwargs):
# TODO: how to use kwargs?
# def query_usage_report(self, arg, id=None, metric='cpu',
# metric_param='total',
# statistic='avg', period=5,
# timestamp_from=None, timestamp_to=None,
# **kwargs):
"""statistic is STATISTIC enum
period default=5 minutes
time_to default=0(now)"""
"""
{
'id': 'instance00001',
'metric': 'network',
'metric_param': 'vnet0',
'statistic': 'sum',
'period': 5,
'timestamp_from': '2012-02-20T12:12:12',
'timestamp_to': '2012-02-22T12:12:12',
}
"""
# usage_report = dict()
# datetime_from = iso8601.parse_date(timestamp_from)
# datetime_to = iso8601.parse_date(timestamp_to)
# # TODO: implement
# return {'data': usage_report}
row_id = args['id']
cf_str = args['metric']
scf_str = args.setdefault("metric_param", "total")
statistic = args['statistic']
period = int(args['period'])
timestamp_from = args['timestamp_from']
timestamp_to = args['timestamp_to']
# time_from = iso8601.parse_date(timestamp_from)
# time_from = int(time.mktime(time_from.timetuple()))
time_from = parse(timestamp_from)
time_from = int(time.mktime(time_from.timetuple()))
time_to = int(utils.utcnow_ts())
if not timestamp_to is None:
# time_to = iso8601.parse_date(timestamp_to)
# time_to = int(time.mktime(time_to.timetuple()))
time_to = parse(timestamp_to)
time_to = int(time.mktime(time_to.timetuple()))
bufkey = str([row_id, cf_str, scf_str,
statistic, period, time_from, time_to])
print "query_usage_report time range:", time_from, time_to
if self.buf.hit_test(bufkey):
print "buffer hit:", bufkey
return self.buf.get_buf(bufkey)
ret_len = 0
(rs, count, all_data) = self.get_data(row_id, cf_str, scf_str,
time_from, time_to)
if not rs is None and count > 0:
buf = self.analyize_data(rs, 1, statistic)
ret = self.analyize_data(buf, period, statistic, show=True)
if ret is None:
ret_len = 0
else:
ret = OrderedDict(sorted(ret.items(), key=lambda t: t[0]))
ret_len = len(ret)
print ret_len, "result."
else:
print "no result."
ret = None
ret_len = 0
result = ret, ret_len, all_data
if (not result is None and utils.utcnow_ts() - time_to > 120):
self.buf.cleanup()
self.buf.save(bufkey, result)
return result
def statistic(self, row_id, cf_str, scf_str,
statistic, period=5, time_from=0, time_to=0):
"""statistic is STATISTIC enum
period default=5 minutes
time_to default=0(now)"""
if (not isinstance(row_id, unicode) \
or not isinstance(cf_str, unicode) \
or not isinstance(scf_str, unicode) \
or not isinstance(statistic, basestring) \
or not isinstance(period, int) \
or not isinstance(time_from, int) \
or not isinstance(time_to, int)):
print 'param types error'
return None, 0, True
bufkey = str([row_id, cf_str, scf_str,
statistic, period, time_from, time_to])
if self.buf.hit_test(bufkey):
return self.buf.get_buf(bufkey)
ret_len = 0
(rs, count, all_data) = self.get_data(row_id, cf_str, scf_str,
time_from, time_to)
if not rs is None and count > 0:
buf = self.analyize_data(rs, 1, statistic)
ret = self.analyize_data(buf, period, statistic, show=True)
if ret is None:
ret_len = 0
else:
ret = OrderedDict(sorted(ret.items(), key=lambda t: t[0]))
ret_len = len(ret)
print ret_len, "result."
else:
print "no result."
ret = None
ret_len = 0
result = ret, ret_len, all_data
self.buf.save(bufkey, result)
return result
