def get_rows(task_id, date):
original_data = data_db.OriginalData()
rows = original_data.find({"@create": date, "@task": int(task_id)})
rows = list(rows)
if len(rows) == 0:
logging.fatal("[ERROR]original data not found!")
exit(-1)
return rows
def copy_to_summary(task_id, from_date, to_date):
daily_summary_data = data_db.DailySummaryData()
daily_summary_data.remove({"@task": task_id, "@date": to_date})
original_data = data_db.OriginalData()
lines = original_data.find({"@task": task_id, "@create": from_date})
for line in lines:
del line["_id"]
line["@date"] = to_date
daily_summary_data.insert(line)
def get_summary_date(task_id, date):
u"""
更新指定的date，以及summary表中已有的，且比原始表中下一个日期小的日期
达到的效果：
1、对于例行指标更新最新指标情况下，只更新当天的summary
2、对于非例行指标更新最新指标情况下，更新截止目前调度工具已更新的所有summary
3、对于存在DATEA、DATEB两天指标情况下，更新DATEA，更新DATEA->DATEB之间的所有summary
:param task_id:
:param date:
:return:
"""
date_list = [date]
original_data = data_db.OriginalData()
last_date = original_data.find({
"@create": {
"$gt": date
},
"@task": task_id
}, {"@create": 1})
last_date.sort([("@create", 1)])
last_date = last_date[:1]
last_date = list(last_date)
if len(last_date) > 0:
last_date = last_date[0]["@create"]
else:
# 若无更新的数据，则至明天
today = datetime.date.today()
tomorrow = today + datetime.timedelta(days=1)
last_date = tomorrow.strftime("%Y%m%d")
daily_summary_data = data_db.DailySummaryData()
extend_date_list = daily_summary_data.find({
"@date": {
"$gt": date,
"$lt": last_date
},
"@task": task_id
})
extend_date_list = extend_date_list.distinct("@date")
date_list.extend(extend_date_list)
return date_list
def save_index(path, task, date):
sub_project = task.sub_project_id
system_key = {
"@task": task.id,
"@create": date,
"@subProject": sub_project
}
original_data = data_db.OriginalData()
original_data.remove(system_key)
fp = open(path)
for line in fp:
line = line.rstrip("\r\n").decode("utf-8")
try:
json_line = json.loads(line)
except:
logging.error("json error:%s" % line)
if base.check_line(json_line, MUST_KEYS):
json_line.update(system_key)
try:
original_data.insert(json_line)
except:
tools.log("%s" % json_line)
raise
def get_insert_date(task_id, date):
u"""
从原始表中查找需要导入summary表的数据日期
:param task_id:
:param date:
:return:
"""
original_data = data_db.OriginalData()
insert_date = original_data.find(
{
"@task": task_id,
"@create": {
"$lte": date
}
}, {"@create": 1})
insert_date.sort([("@create", -1)])
insert_date = insert_date[:1]
insert_date = list(insert_date)
if len(insert_date) > 0:
insert_date = insert_date[0]["@create"]
else:
insert_date = None
return insert_date