Exemple #1
0
 def update_od_imp(now, minute_duration):
     raw_imp_info = Imp.get_raw_od_imp(now, minute_duration)
     for raw_imp in raw_imp_info:
         order_item_id = raw_imp.get('_id').get('order_item_id')
         time = '%s' % (raw_imp.get('_id').get('time') + datetime.timedelta(hours=8))
         impression_count = raw_imp.get('imp')
         Imp.order_item_imp.setdefault(order_item_id, {})
         Imp.order_item_imp.get(order_item_id)[time] = impression_count
     LogUtils.info("Finish getting the each order item impression info.")
 def update_outlier_detect():
     for comb in Comb.comb:
         alias = comb.get('alias')
         imp_info = Imp.comb_imp.get(alias)
         counts = []
         minutes = []
         for time in imp_info.keys():
             counts.append(imp_info.get(time).get('impression_count'))
             minutes.append(time)
         AutoDetection.points.setdefault(alias, [])
         AutoDetection.points[alias] = AutoDetection.outlier_detect(counts, minutes)
         LogUtils.info("Finish the auto detection on %s" % alias)
Exemple #3
0
    def update_all_imp():
        now = datetime.datetime.now().replace(second=0, microsecond=0) - datetime.timedelta(hours=8)

        threads = []
        od_thread = threading.Thread(target=Imp.update_od_imp, args=(now, 1440))
        combs_thread = threading.Thread(target=Imp.update_combs_imp, args=(now, 1440))

        od_thread.start()
        combs_thread.start()

        threads.append(od_thread)
        threads.append(combs_thread)

        ThreadUtils.wait_until_done(threads)
        LogUtils.info("Finish the data init process!")
Exemple #4
0
    def update_comb_imp(comb, order_items, now, minute_duration):
        alias = comb.get('alias')
        LogUtils.info("Updating the %s impression data..." % alias)
        order_item_list = []
        for order_item_info in order_items.get(alias).get(OrderItemList.order_item):
            order_item_list.append('%s' % order_item_info.get('order_item_id'))
        imp_info = Imp.get_raw_comb_imp(now, order_item_list, minute_duration)
        for order_item_info in imp_info:
            impression_count = order_item_info['imp_count']
            time = '%s' % (order_item_info['_id']['time'] + datetime.timedelta(hours=8))
            order_item_count = order_item_info['od_count']
            Imp.comb_imp.setdefault(alias, {})
            Imp.comb_imp.get(alias).setdefault(time, {})
            Imp.comb_imp.get(alias).get(time)['impression_count'] = impression_count
            Imp.comb_imp.get(alias).get(time)['order_item_count'] = order_item_count

        LogUtils.info("Finish updating the %s impression data." % alias)
Exemple #5
0
    def update_latest_imp():
        now = datetime.datetime.now().replace(second=0, microsecond=0) - datetime.timedelta(hours=8)

        LogUtils.info("Last updated Beijing time: %s" % (now + datetime.timedelta(hours=8)))
        threads = []
        od_thread = threading.Thread(target=Imp.update_od_imp, args=(now, 15))
        combs_thread = threading.Thread(target=Imp.update_combs_imp, args=(now, 15))

        threads.append(od_thread)
        threads.append(combs_thread)

        od_thread.start()
        combs_thread.start()

        ThreadUtils.wait_until_done(threads)

        Imp.delete_obs_data(Imp.order_item_imp, now)
        Imp.delete_obs_data(Imp.comb_imp, now)
Exemple #6
0
    def update_dict():
        comb_dict = Dict()
        time = datetime.datetime.now()
        raw_log = Dict.get_raw_info(time.date())
        for log in raw_log:
            if log[comb_dict.param_index['order_item_id']] not in comb_dict.dict_od_info:
                comb_dict.add_od(log)
            else:
                comb_dict.append_od(log)

        for value in comb_dict.dict_od_info.values():
            comb_dict.list_od_info.append(value)

        for info in comb_dict.list_od_info:
            MongoDbUtils.get_dict_collection().replace_one({"order_item_id": info['order_item_id']}, info, True)

        MongoDbUtils.get_dict_collection().delete_many(
            {"end_date": time.replace(hour=0, minute=0, second=0, microsecond=0) - datetime.timedelta(1)}
        )

        size = MongoDbUtils.get_dict_collection().count()
        LogUtils.info("Finish the order item dict synchronization successfully, dict size = %s" % size)
 def update_ods():
     order_items = OrderItems()
     all_order_item_list = list(MongoDbUtils.get_dict_collection().find())
     combs = Comb.comb
     for comb in combs:
         order_item_dict = dict()
         order_item_dict[OrderItemList.name] = comb[CombFields.name]
         order_item_dict[OrderItemList.alias] = comb[CombFields.alias]
         if len(comb.get(CombFields.condition, [])) == 0:
             order_item_list = []
             for order_item in all_order_item_list:
                 order_item_list.append(order_item)
             order_item_dict[OrderItemList.order_item] = order_item_list
         elif len(comb.get(CombFields.condition, [])) == 1:
             order_item_dict[OrderItemList.order_item] = order_items.filter_one_condition(
                 all_order_item_list, comb.get(CombFields.condition)[0]
             )
         else:
             order_item_dict[OrderItemList.order_item] = order_items.filter_multiple_conditions(
                 all_order_item_list, comb.get(CombFields.condition)
             )
         LogUtils.info("Finish generating the order item list %s, the size = %d" % (comb.get('alias'), len(order_item_dict[OrderItemList.order_item])))
         OrderItems.ods[comb[CombFields.alias]] = order_item_dict