def insert_log(self, ch, method, properties, body):
self.insert_datas.append(json.loads(str(body, encoding="utf-8")))
self.delivery_tags.append(method.delivery_tag)
is_ack = False
global insert_interval_time_stamp
now_time_stamp = Time.getNowTimeStamp()
interval_time = now_time_stamp - insert_interval_time_stamp
if len(
self.insert_datas
) >= init.MAX_INSERT_COUNT or interval_time > init.INSERT_INETRVAL_TIME:
insert_interval_time_stamp = Time.getNowTimeStamp()
is_ack = True
try:
L.info("will insert data count: is %f", len(self.insert_datas))
mongo_instance = Mongo.getInstance(table="YXYBB_interface",
ds="YXYBB")
collection = mongo_instance.getCollection()
collection.insert_many(self.insert_datas)
mongo_instance.close()
self.insert_datas = []
except Exception as e:
L.warning("insert_log Exception %s", e)
if is_ack is True:
for delivery_tag in self.delivery_tags:
ch.basic_ack(delivery_tag=delivery_tag)
self.delivery_tags = []
pass
def line_test(self, data):
_PR = PR.getInstance()
_title_text = data['title_text'] # 标题
_step_count = data["step_count"] # 横坐标点
_step = data['step'] # 步长（单位:min）
_type = data['type'] # 绘图类型
# 保宝网的点击量和保宝网的观看人数统计的折线图 怎么处理？？？ 项目 类型
_legend_infos = data['legend_infos'] # 数据项信息
print(_legend_infos)
_legend_datas = _legend_infos.keys() # 数据项名称
_search_filter_infos = {}
for _legend_data in _legend_datas:
_project_name = _legend_infos[_legend_data]['project_name'] # 项目名称
_statistic_type = _legend_infos[_legend_data]['statistic_type'] # 统计类型
_statistic_name = _legend_infos[_legend_data]['statistic_name'] # 统计名称
_filter_infos = _legend_infos[_legend_data]['filter_infos'] # 过滤条件
print(_filter_infos)
ds = logic.project_ds_info[_project_name] # 查询数据源
table = _project_name + "_" + _statistic_type # YXYBB_interface
self_mongo_instance = Mongo.getInstance(table=table, ds=ds)
self_collection = self_mongo_instance.getCollection()
_search_filter_infos[_legend_data] = {
"project_name": _project_name,
"self_collection": self_collection, # 连接额外数据源
"filter_infos": _filter_infos, # 过滤机制
"statistic_type": _statistic_type, # 统计类型
"statistic_name": _statistic_name # 统计名称
}
_result = Line.getInsatnce(search_filter_infos=_search_filter_infos, _step=_step, _step_count=_step_count, _title_text=_title_text, _type=_type).getLineChartData()
self_mongo_instance.close()
_PR.setResult(_result)
return _PR.getPRBytes()
def init_DB_ds(self):
path = P.confDirPath + os.sep + "DB.json"
DB_infos = J.readFile(path)
mongo_instance = Mongo.getInstance(table="project_ds", ds='base')
collection = mongo_instance.getCollection()
collection.remove({}) # 先删除表中所有数据
datas = []
for key in DB_infos.keys():
value = DB_infos[key]['dbname']
data = {}
data["ds_code"] = key
data["project"] = value
datas.append(data)
L.info("init DB_ds , insert data:" + key + ":" + value)
collection.insert_many(datas)
mongo_instance.close()
pass
def getTasks(self):
'''
"id":"主键",
"project_name":"项目名称",
"createtime":"数据创建时间",
"updatetime":"数据更新时间",
"statistical_type":"统计类型",
"statistical_lastTime":"最后一次统计时间"，
"statistical_startTime":"起始统计时间",
"statistical_step":"统计频率",
"statistical_name“:"统计名称"
'''
if self.queue is None:
L.error("task publish not the right queue")
return None
tasks = []
task_mongo_instance = Mongo.getInstance(table="statistical_item",
ds='base')
task_collection = task_mongo_instance.getCollection()
statistical_datas = task_collection.find({}).sort(
"statistical_step", 1)
statistical_step_min = 0
statistical_step_datas = []
for statistical_data in statistical_datas.sort("statistical_step"):
statistical_step = statistical_data['statistical_step']
if statistical_step_min == statistical_step or statistical_step_min is 0:
statistical_step_min = statistical_step
statistical_step_datas.append(statistical_data)
else:
task = {
"method": self.statistical_compute,
"pars": statistical_step_datas.copy()
}
tasks.append(task)
statistical_step_min = statistical_step
del statistical_step_datas[:]
statistical_step_datas.append(statistical_data)
task = {
"method": self.statistical_compute,
"pars": statistical_step_datas.copy()
}
tasks.append(task)
return tasks
def line_test(self, data):
_PR = PR.getInstance()
_title_text = data['title_text'] # 标题
_step_count = data["step_count"] # 横坐标点
_step = data['step'] # 步长（单位:min）
_type = data['type'] # 绘图类型
# 保宝网的点击量和保宝网的观看人数统计的折线图 怎么处理？？？ 项目 类型
_legend_infos = data['legend_infos'] # 数据项信息
print(_legend_infos)
_legend_datas = _legend_infos.keys() # 数据项名称
_search_filter_infos = {}
for _legend_data in _legend_datas:
_project_name = _legend_infos[_legend_data]['project_name'] # 项目名称
_statistic_type = _legend_infos[_legend_data][
'statistic_type'] # 统计类型
_statistic_name = _legend_infos[_legend_data][
'statistic_name'] # 统计名称
_filter_infos = _legend_infos[_legend_data]['filter_infos'] # 过滤条件
print(_filter_infos)
ds = logic.project_ds_info[_project_name] # 查询数据源
table = _project_name + "_" + _statistic_type # YXYBB_interface
self_mongo_instance = Mongo.getInstance(table=table, ds=ds)
self_collection = self_mongo_instance.getCollection()
_search_filter_infos[_legend_data] = {
"project_name": _project_name,
"self_collection": self_collection, # 连接额外数据源
"filter_infos": _filter_infos, # 过滤机制
"statistic_type": _statistic_type, # 统计类型
"statistic_name": _statistic_name # 统计名称
}
_result = Line.getInsatnce(search_filter_infos=_search_filter_infos,
_step=_step,
_step_count=_step_count,
_title_text=_title_text,
_type=_type).getLineChartData()
self_mongo_instance.close()
_PR.setResult(_result)
return _PR.getPRBytes()
def callback(self, ch, method, properties, body):
print(" [x] Received %r" % body)
print(str(body, encoding="utf-8"))
collection = Mongo.getInstance(table="YXYBB_interface",
dbname="YXYBB").collection
collection.insert(json.loads(str(body, encoding="utf-8")))
def statistical_deal(self, pars):
while True:
sleep_time = 60 * 60
for par in pars:
_id = par['_id']
_item_mongo_instnce = Mongo.getInstance(
table="statistical_item")
_item_conllection = _item_mongo_instnce.getCollection()
_item_bo = statistical_item_BO.getInstance()
_item_bo.setId(_id)
_item_filter = Filter.getInstance().filter(
"_id", _id, DBCODE.EQ)
_item_info = _item_conllection.find_one(
_item_filter.filter_json())
statistical_step = _item_info['statistical_step']
sleep_time = statistical_step * 60
project_name = _item_info['project_name']
statistical_type = _item_info['statistical_type']
if "statistical_name" in _item_info.keys():
statistical_name = _item_info['statistical_name']
else:
statistical_name = None
statistical_start_time = _item_info['statistical_start_time']
times = Time.getComputeTimes(start_time=statistical_start_time,
step=statistical_step)
ds = logic.project_ds_info[project_name]
table = project_name + "_" + statistical_type
# 数据源数据，用于统计数据
statistical_mongo_instance = Mongo.getInstance(table=table,
ds=ds)
statistical_mongo_collection = statistical_mongo_instance.getCollection(
)
documents = []
lastTime = None
L.debug("compute step is %d s", statistical_step * 60)
for i in range(1, len(times)):
lastTime = times[i]
_f = Filter.getInstance()
_f.filter("type", statistical_type, DBCODE.EQ)
_f.filter("project", project_name, DBCODE.EQ)
if statistical_name is not None:
_f.filter("name", statistical_name, DBCODE.EQ)
_f.filter("createtime", times[i - 1], DBCODE.GT)
_f.filter("createtime", times[i], DBCODE.LTE)
_filter = _f.filter_json()
count = statistical_mongo_collection.find(_filter).count()
document_bo = statistic_res_BO.getInstance()
document_bo.setStatistical_project(project_name)
document_bo.setStatistical_time(times[i])
document_bo.setStatistical_count(count)
document_bo.setStatistical_step(statistical_step)
document_bo.setStatistical_type(statistical_type)
if statistical_name is not None:
document_bo.setStatistical_name(statistical_name)
documents.append(document_bo.json())
if len(documents) > init.MAX_INSERT_COUNT:
res_mongo_instance = Mongo.getInstance(
table=BO.BASE_statistic_res)
res_collection = res_mongo_instance.getCollection()
res_collection.insert_many(
documents=documents) # 将结果插入到结果表中,防止爆了
documents = []
res_mongo_instance.close()
if len(documents) > 0:
res_mongo_instance = Mongo.getInstance(
table=BO.BASE_statistic_res)
res_collection = res_mongo_instance.getCollection()
res_collection.insert_many(
documents=documents) # 将结果插入到结果表中
documents = []
res_mongo_instance.close()
else:
L.debug("statistical_deal ,not get the insert data")
# 去更新statistical_item表
if lastTime is not None:
_item_bo_1 = statistical_item_BO.getInstance()
_item_bo_1.setStatistical_start_time(lastTime)
_item_conllection.update_one(_item_filter.filter_json(),
_item_bo_1.update_json)
else:
L.debug("statistical_deal ,not get last time")
# 关闭数据库连接
_item_mongo_instnce.close()
L.debug("thread will sleep %s s", sleep_time)
time.sleep(sleep_time)
self.queue = queue.Queue(self.step_type_count)
self.start() # 启动队列
for task in tasks:
self.queue.put(task, block=True, timeout=None) # 向队列中添加内容
# self.queue.join() # 等待所有任务完成
L.info("item compute task add finished ")
# 计算全部定时任务内容代码开始启动 (启动一个线程来进行处理)
def start_init(self):
t = threading.Thread(target=self.product_consume)
t.start()
if __name__ == "__main__":
item_bo = statistical_item_BO.getInstance()
collection = Mongo.getInstance(
table=BO.BASE_statistical_item).getCollection()
datas = []
data_1 = item_bo.setProject_name("YXYBB").setStatistical_type(
"interface").setStatistical_step(1).setStatistical_start_time(
"2017-05-18 00:00:00.000").json
data_2 = item_bo.setProject_name("YXYBB").setStatistical_type(
"interface").setStatistical_step(5).setStatistical_start_time(
"2017-05-18 00:00:00.000").json
data_3 = item_bo.setProject_name("YXYBB").setStatistical_type(
"interface").setStatistical_step(30).setStatistical_start_time(
"2017-05-18 00:00:00.000").json
data_4 = item_bo.setProject_name("YXYBB").setStatistical_type(
"interface").setStatistical_step(60).setStatistical_start_time(
"2017-05-18 00:00:00.000").json
data_5 = item_bo.setProject_name("YXYBB").setStatistical_type(
"interface").setStatistical_step(
def stop(self):
self.thread_stop = True
if __name__ == "__main__":
# q = queue.Queue(3)
# worker = worker(q)
# worker.start()
# q.put(["produce one cup!", 1], block=True, timeout=None) # 产生任务消息
# q.put(["produce one desk!", 2], block=True, timeout=None)
# q.put(["produce one apple!", 3], block=True, timeout=None)
# q.put(["produce one banana!", 4], block=True, timeout=None)
# q.put(["produce one bag!", 5], block=True, timeout=None)
# print("***************leader:wait for finish!")
# q.join() # 等待所有任务完成
# print("***************leader:all task finished!")
collection = Mongo.getInstance(table=BO.BASE_statistic_res).getCollection()
# count = collection.find({'statistical_time': {'$gt': '2017-05-16 11:25:00', '$lte': '2017-05-23 10:05:00'}, 'statistical_step': 1, 'statistical_type': 'interface', 'statistical_project': 'YXYBB','statistical_name':None}).count()
count = collection.find({
'statistical_step': 1,
'statistical_project': 'YXYBB',
'statistical_time': {
'$lte': '2017-05-23 11:42:00',
'$gt': '2017-05-16 13:02:00'
},
'statistical_type': 'interface',
'statistical_name': None
}).count()
print(count)
#!/usr/bin/env python
# !-*- coding:utf-8 -*-
from bin.until import Mongo
project_ds_info = {}
for ds_info in Mongo.getInstance(table="project_ds").getCollection().find():
project_ds_info[ds_info["project"]] = ds_info["ds_code"]
class Service_logic(object):
def logic(self, data):
_PR = PR.getInstance()
_data = {
"title": {
"text": 'ECharts 入门示例'
},
"tooltip": {},
"legend": {
"data": ['销量']
},
"xAxis": {
"data": ["衬衫", "羊毛衫", "雪纺衫", "裤子", "高跟鞋", "袜子"]
},
"yAxis": {},
"series": [{
"name": '销量',
"type": 'bar',
"data": [5, 20, 36, 10, 10, 200]
}]
}
_PR.setResult(_data)
print(_PR.getResult())
# L.debug(_data)
# L.info(_data)
# L.warn(_data)
# L.error(_data)
# L.critical(_data)
return _PR.getPRBytes()
def xx(self, data):
_PR = PR.getInstance()
chart = Echart('GDP', 'This is a fake chart')
chart.use(Bar('China', [2, 3, 4, 5]))
chart.use(Legend(['GDP']))
chart.use(Axis('category', 'bottom', data=['Nov', 'Dec', 'Jan',
'Feb']))
_chart = chart.json
print(_chart)
_chart["tooltip"] = {}
_PR.setResult(_chart)
return _PR.getPRBytes()
collection = Mongo.getInstance("statistics").collection
def line_test(self, data):
_PR = PR.getInstance()
_title_text = data['title_text'] # 标题
_step_count = data["step_count"] # 横坐标点
_step = data['step'] # 步长（单位:min）
_type = data['type'] # 绘图类型
# 保宝网的点击量和保宝网的观看人数统计的折线图 怎么处理？？？ 项目 类型
_legend_infos = data['legend_infos'] # 数据项信息
print(_legend_infos)
_legend_datas = _legend_infos.keys() # 数据项名称
_search_filter_infos = {}
for _legend_data in _legend_datas:
_project_name = _legend_infos[_legend_data]['project_name'] # 项目名称
_statistic_type = _legend_infos[_legend_data][
'statistic_type'] # 统计类型
_statistic_name = _legend_infos[_legend_data][
'statistic_name'] # 统计名称
_filter_infos = _legend_infos[_legend_data]['filter_infos'] # 过滤条件
print(_filter_infos)
ds = logic.project_ds_info[_project_name] # 查询数据源
table = _project_name + "_" + _statistic_type # YXYBB_interface
self_mongo_instance = Mongo.getInstance(table=table, ds=ds)
self_collection = self_mongo_instance.getCollection()
_search_filter_infos[_legend_data] = {
"project_name": _project_name,
"self_collection": self_collection, # 连接额外数据源
"filter_infos": _filter_infos, # 过滤机制
"statistic_type": _statistic_type, # 统计类型
"statistic_name": _statistic_name # 统计名称
}
_result = Line.getInsatnce(search_filter_infos=_search_filter_infos,
_step=_step,
_step_count=_step_count,
_title_text=_title_text,
_type=_type).getLineChartData()
self_mongo_instance.close()
_PR.setResult(_result)
return _PR.getPRBytes()
def getLineChartData(self):
series = []
_legend_datas = []
for key in self._search_filter_infos:
_legend_data = key
_legend_datas.append(_legend_data)
_search_filter_info = self._search_filter_infos[key]
_project = _search_filter_info['project_name']
self_collection = _search_filter_info['self_collection']
_filter_infos = _search_filter_info['filter_infos']
_statistic_type = _search_filter_info['statistic_type']
_statistic_name = _search_filter_info['statistic_name']
self.start_time = Time.getStartTime(
step=self._step, step_count=self._step_count) # 获取起始时间
is_search_db = False
for _filter_info in _filter_infos:
key = _filter_info['key']
relation = _filter_info['relation']
value = _filter_info['value']
if key == 'time' and (relation == DBCODE.GT
or relation == DBCODE.GTE):
self.start_time = value # 过滤条件中的起始时间
elif key == 'time' and (relation == DBCODE.LTE
or relation == DBCODE.LT):
self.end_time = value # 过滤条件中的终止时间
else:
is_search_db = True
times = Time.getComputeTimes(start_time=self.start_time,
end_time=self.end_time,
step=self._step)
series_data = [] # y轴上的信息
if is_search_db is True: # 多条件查询
_self_filter = Filter.getInstance()
_self_filter.filter("project", _project, DBCODE.EQ)
_self_filter.filter("type", _statistic_type, DBCODE.EQ)
for _filter_info in _filter_infos:
if _filter_info['key'] != 'time':
_self_filter.filter(_filter_info['key'],
_filter_info['value'],
_filter_info['relation'])
for i in range(len(times) - 1):
_self_filter.filter("createtime", times[i], DBCODE.GT)
_self_filter.filter("createtime", times[i + 1], DBCODE.LTE)
_filter = _self_filter.filter_json()
count = self_collection.find(_filter).count()
series_data.append(count)
else:
# 计划分批次查询
res_collection = Mongo.getInstance(
table=BO.BASE_statistic_res).getCollection()
res_filter = Filter.getInstance()
res_filter.filter("statistical_time", times[0], DBCODE.GT)
res_filter.filter("statistical_time", times[-1], DBCODE.LTE)
res_filter.filter("statistical_step", self._step, DBCODE.EQ)
res_filter.filter("statistical_type", _statistic_type,
DBCODE.EQ)
res_filter.filter("statistical_project", _project, DBCODE.EQ)
if Data.isNone(_statistic_name):
_statistic_name = None
res_filter.filter("statistical_name", _statistic_name,
DBCODE.EQ)
print(res_filter.filter_json())
ress = res_collection.find(res_filter.filter_json()).sort(
"statistical_time", -1) # 计算前半部分值
self._step_count = len(times) - 1
series_data = Data.getD4tArr(len=self._step_count,
default_value=0) # 坐标轴上的值
# 先来尝试组合数据，发现数据无法组合完整时，补充数据
i = 0
for res in ress:
if i == 0 and ress.count() != (
len(times) -
1) and res['statistical_time'] != times[-1]:
# 重新补录一个值
_self_filter = Filter.getInstance()
if not Data.isNone(_statistic_name):
_self_filter.filter("name", _statistic_name,
DBCODE.EQ)
_self_filter.filter("project", _project, DBCODE.EQ)
_self_filter.filter("type", _statistic_type, DBCODE.EQ)
_self_filter.filter("createtime", times[-2], DBCODE.GT)
_self_filter.filter("createtime", times[-1],
DBCODE.LTE)
_filter = _self_filter.filter_json()
count = self_collection.find(_filter).count()
series_data[i] = count
series_data[i + 1] = res['statistical_count']
i = i + 2
else:
series_data[i] = res['statistical_count']
i = i + 1
series_data.reverse()
xAxis_data = times[1:] # 横坐标轴信息[] 时间信息 去掉首要点
serie = {
"name": _legend_data,
"type": self._type,
"showSymbol": False,
"smooth": True,
# "stack": '总量',
"data": series_data.copy() # 坐标轴上的值
}
series.append(serie)
_result = {
"title": {
"text": self._title_text
},
"legend": {
"data": _legend_datas.copy()
},
"xAxis": {
"data": xAxis_data.copy()
},
"series": series
}
return _result