def update_od_imp(now, minute_duration):
raw_imp_info = Imp.get_raw_od_imp(now, minute_duration)
for raw_imp in raw_imp_info:
order_item_id = raw_imp.get('_id').get('order_item_id')
time = '%s' % (raw_imp.get('_id').get('time') + datetime.timedelta(hours=8))
impression_count = raw_imp.get('imp')
Imp.order_item_imp.setdefault(order_item_id, {})
Imp.order_item_imp.get(order_item_id)[time] = impression_count
LogUtils.info("Finish getting the each order item impression info.")
def update_outlier_detect():
for comb in Comb.comb:
alias = comb.get('alias')
imp_info = Imp.comb_imp.get(alias)
counts = []
minutes = []
for time in imp_info.keys():
counts.append(imp_info.get(time).get('impression_count'))
minutes.append(time)
AutoDetection.points.setdefault(alias, [])
AutoDetection.points[alias] = AutoDetection.outlier_detect(counts, minutes)
LogUtils.info("Finish the auto detection on %s" % alias)
def update_all_imp():
now = datetime.datetime.now().replace(second=0, microsecond=0) - datetime.timedelta(hours=8)
threads = []
od_thread = threading.Thread(target=Imp.update_od_imp, args=(now, 1440))
combs_thread = threading.Thread(target=Imp.update_combs_imp, args=(now, 1440))
od_thread.start()
combs_thread.start()
threads.append(od_thread)
threads.append(combs_thread)
ThreadUtils.wait_until_done(threads)
LogUtils.info("Finish the data init process!")
def update_comb_imp(comb, order_items, now, minute_duration):
alias = comb.get('alias')
LogUtils.info("Updating the %s impression data..." % alias)
order_item_list = []
for order_item_info in order_items.get(alias).get(OrderItemList.order_item):
order_item_list.append('%s' % order_item_info.get('order_item_id'))
imp_info = Imp.get_raw_comb_imp(now, order_item_list, minute_duration)
for order_item_info in imp_info:
impression_count = order_item_info['imp_count']
time = '%s' % (order_item_info['_id']['time'] + datetime.timedelta(hours=8))
order_item_count = order_item_info['od_count']
Imp.comb_imp.setdefault(alias, {})
Imp.comb_imp.get(alias).setdefault(time, {})
Imp.comb_imp.get(alias).get(time)['impression_count'] = impression_count
Imp.comb_imp.get(alias).get(time)['order_item_count'] = order_item_count
LogUtils.info("Finish updating the %s impression data." % alias)
def update_latest_imp():
now = datetime.datetime.now().replace(second=0, microsecond=0) - datetime.timedelta(hours=8)
LogUtils.info("Last updated Beijing time: %s" % (now + datetime.timedelta(hours=8)))
threads = []
od_thread = threading.Thread(target=Imp.update_od_imp, args=(now, 15))
combs_thread = threading.Thread(target=Imp.update_combs_imp, args=(now, 15))
threads.append(od_thread)
threads.append(combs_thread)
od_thread.start()
combs_thread.start()
ThreadUtils.wait_until_done(threads)
Imp.delete_obs_data(Imp.order_item_imp, now)
Imp.delete_obs_data(Imp.comb_imp, now)
def update_dict():
comb_dict = Dict()
time = datetime.datetime.now()
raw_log = Dict.get_raw_info(time.date())
for log in raw_log:
if log[comb_dict.param_index['order_item_id']] not in comb_dict.dict_od_info:
comb_dict.add_od(log)
else:
comb_dict.append_od(log)
for value in comb_dict.dict_od_info.values():
comb_dict.list_od_info.append(value)
for info in comb_dict.list_od_info:
MongoDbUtils.get_dict_collection().replace_one({"order_item_id": info['order_item_id']}, info, True)
MongoDbUtils.get_dict_collection().delete_many(
{"end_date": time.replace(hour=0, minute=0, second=0, microsecond=0) - datetime.timedelta(1)}
)
size = MongoDbUtils.get_dict_collection().count()
LogUtils.info("Finish the order item dict synchronization successfully, dict size = %s" % size)
def update_ods():
order_items = OrderItems()
all_order_item_list = list(MongoDbUtils.get_dict_collection().find())
combs = Comb.comb
for comb in combs:
order_item_dict = dict()
order_item_dict[OrderItemList.name] = comb[CombFields.name]
order_item_dict[OrderItemList.alias] = comb[CombFields.alias]
if len(comb.get(CombFields.condition, [])) == 0:
order_item_list = []
for order_item in all_order_item_list:
order_item_list.append(order_item)
order_item_dict[OrderItemList.order_item] = order_item_list
elif len(comb.get(CombFields.condition, [])) == 1:
order_item_dict[OrderItemList.order_item] = order_items.filter_one_condition(
all_order_item_list, comb.get(CombFields.condition)[0]
)
else:
order_item_dict[OrderItemList.order_item] = order_items.filter_multiple_conditions(
all_order_item_list, comb.get(CombFields.condition)
)
LogUtils.info("Finish generating the order item list %s, the size = %d" % (comb.get('alias'), len(order_item_dict[OrderItemList.order_item])))
OrderItems.ods[comb[CombFields.alias]] = order_item_dict
