def get_banner_events_iter(banner_id, timestamp): """ Fetch all banner documents from the database. :param banner_id: Banner identifier. :param timestamp: Time in seconds since the epoch, used for getting the full hour timestamp. :return: Iterable events collection (QueryIterator) """ timestamp = common_utils.timestamp2hour(timestamp) collection = yield db.get_event_collection() defer.returnValue(QueryIterator(collection.find({ 'banner_id': banner_id, 'timestamp': common_utils.timestamp2hour(timestamp) }, cursor=True)))
def update_user_score(campaign_id, timestamp, user_id, score): """ Update user score with new score and timestamp, per campaign. :param campaign_id: :param timestamp: :param user_id: :param score: :return: """ timestamp = common_utils.timestamp2hour(timestamp) collection = yield db.get_user_score_collection() return_value = yield collection.replace_one({ 'campaign_id': campaign_id, 'timestamp': timestamp, 'user_id': user_id }, { 'campaign_id': campaign_id, 'timestamp': timestamp, 'user_id': user_id, 'score': score }, upsert=True) defer.returnValue(return_value)
def step_impl(context, timestamp): def test_doc(doc): assert doc is not None assert 'timestamp' in doc assert doc['timestamp'] <= timestamp timestamp = timestamp2hour(timestamp) last_round_doc = utils.get_payment_round(timestamp) last_round_doc.addCallback(test_doc)
def delete_payment_round(timestamp): """ Remove the payment round from collection. :param timestamp: :return: """ timestamp = common_utils.timestamp2hour(timestamp) collection = yield db.get_payment_rounds_collection() return_value = yield collection.delete_many({'timestamp': timestamp}) defer.returnValue(return_value)
def delete_user_scores(campaign_id, timestamp): """ :param campaign_id: :param timestamp: :return: """ timestamp = common_utils.timestamp2hour(timestamp) collection = yield db.get_user_score_collection() return_value = yield collection.delete_many({'campaign_id': campaign_id, 'timestamp': timestamp}) defer.returnValue(return_value)
def get_payment_round(timestamp): """ Fetch a payment round document from the database. :param timestamp: Timestamp (epoch, seconds, full hour) for this request. :return: Payment round document. """ timestamp = common_utils.timestamp2hour(timestamp) collection = yield db.get_payment_rounds_collection() return_value = yield collection.find_one({'timestamp': timestamp}) defer.returnValue(return_value)
def get_payments_iter(timestamp): """ Fetch all payment documents from the database. :param timestamp: Timestamp (epoch, seconds, full hour) for this request. :return: Iterable payment information. """ timestamp = common_utils.timestamp2hour(timestamp) collection = yield db.get_payment_collection() defer.returnValue(QueryIterator(collection.find({ 'timestamp': timestamp }, cursor=True)))
def update_payment_round(timestamp): """ Create or update a payment round. :param timestamp: :return: """ timestamp = common_utils.timestamp2hour(timestamp) collection = yield db.get_payment_rounds_collection() round_doc = {'timestamp': timestamp} return_value = yield collection.update(round_doc, round_doc, upsert=True) defer.returnValue(return_value)
def get_sorted_user_score_iter(campaign_id, timestamp, limit): """ :param campaign_id: :param timestamp: :param limit: :return: Descending by score sorted list of user score to limit. """ timestamp = common_utils.timestamp2hour(timestamp) collection = yield db.get_user_score_collection() defer.returnValue(QueryIterator(collection.find({'campaign_id': campaign_id, 'timestamp': timestamp}, sort=txfilter.sort(txfilter.DESCENDING("score")), limit=limit, cursor=True)))
def update_event(event_doc): """ Create or update an event if it doesn't exist. :param event_doc: Event document :return: """ event_doc['timestamp'] = common_utils.timestamp2hour(event_doc['timestamp']) collection = yield db.get_event_collection() return_value = yield collection.replace_one({'event_id': event_doc['event_id']}, event_doc, upsert=True) defer.returnValue(return_value)
def get_distinct_users_from_events(campaign_id, timestamp): """ Fetch distinct user identifiers for this campaign, for this timestamp. :param campaign_id: Campaign identifier. :param timestamp: Time in seconds since the epoch, used for getting the full hour timestamp. :return: List of distinct users ids. """ timestamp = common_utils.timestamp2hour(timestamp) collection = yield db.get_event_collection() return_values = yield collection.distinct(key='user_id', filter={'timestamp': timestamp, 'campaign_id': campaign_id}) defer.returnValue(return_values)
def get_events_per_user_iter(campaign_id, timestamp, uid): """ Fetch all events for this campaign, for this timestamp, for this uid. :param campaign_id: Campaign identifier. :param timestamp: Time in seconds since the epoch, used for getting the full hour timestamp. :param uid: User identifier. :return: Iterable events for the user. """ timestamp = common_utils.timestamp2hour(timestamp) collection = yield db.get_event_collection() defer.returnValue(QueryIterator(collection.find({ 'user_id': uid, 'campaign_id': campaign_id, 'timestamp': timestamp }, cursor=True)))
def test_adpay_task(self): cmp_doc = { "campaign_id": "campaign_id", "time_start": 12345, "time_end": 12347, "max_cpc": 100, "max_cpm": 200, "budget": 1000, "filters": {} } yield db_utils.update_campaign(cmp_doc) timestamp = int(time.time()) - stats_consts.SECONDS_PER_HOUR payment_round = yield db_utils.get_payment_round(timestamp) payment_rounds = yield self.get_payment_rounds() self.assertIsNone(payment_round) self.assertEqual(len(payment_rounds), 0) yield stats_tasks._adpay_task(timestamp) yield stats_tasks._adpay_task(timestamp) payment_round = yield db_utils.get_payment_round(timestamp) payment_rounds = yield self.get_payment_rounds() self.assertEqual(payment_round['timestamp'], common_utils.timestamp2hour(timestamp)) self.assertEqual(len(payment_rounds), 1) yield stats_tasks._adpay_task(timestamp, False) cmp_doc = { "campaign_id": "campaign_id", "time_start": 12345, "time_end": 12347, "max_cpc": 100, "max_cpm": 200, "budget": 1000, "filters": {} } yield db_utils.update_campaign(cmp_doc) yield stats_tasks._adpay_task(0) yield stats_tasks._adpay_task(timestamp + 10000)
def step_impl(context, number, timestamp, event_id): class QueryAnalyzer: def __init__(self, dfr): self.finished = False self.length = 0 dfr.addCallback(self.test_query) def test_query(self, query): assert query is not None assert isinstance(query, utils.QueryIterator) while not self.finished: payment_doc = yield query.next() if not payment_doc: self.finished = True self.length += 1 timestamp = timestamp2hour(timestamp) qa = QueryAnalyzer(utils.get_payments_iter(timestamp)) assert qa.length == int(number)
def update_event_payment(campaign_id, timestamp, event_id, payment, reason): """ Create or update payment information for event. :param campaign_id: Campaign identifier. :param timestamp: Timestamp (epoch, in seconds, full hour) :param event_id: Event identifier :param payment: Payment amount. :param reason: Reason (payment classifier). :return: """ timestamp = common_utils.timestamp2hour(timestamp) collection = yield db.get_payment_collection() return_value = yield collection.replace_one({'event_id': event_id, 'campaign_id': campaign_id}, { 'timestamp': timestamp, 'event_id': event_id, 'payment': payment, 'campaign_id': campaign_id, 'reason': reason }, upsert=True) defer.returnValue(return_value)
def test_timestamp2hour(self): ts = common_utils.timestamp2hour(time.time()) self.assertIs(type(ts), int)
def _adpay_task(timestamp=None, ignore_existing_payment_calculations=False): """ Task calculate payments and update user profiles only once a hour. :param timestamp: Timestamp for which to calculate the payments. :param ignore_existing_payment_calculations: Check first if the payment is already calculated. """ logger = logging.getLogger(__name__) # As recalculate only finished hours, take timestamp from an hour before now. if timestamp is None: yield logger.warning( "No timestamp found for recalculation, using current time.") timestamp = int(time.time()) - 3600 timestamp = common_utils.timestamp2hour(timestamp) nice_period_start = datetime.fromtimestamp(timestamp) nice_period_end = datetime.fromtimestamp(timestamp) + timedelta( seconds=3600) if not ignore_existing_payment_calculations: last_round_doc = yield db_utils.get_payment_round(timestamp) if last_round_doc is not None: yield logger.warning( "Payment already calculated for {0} - {1} (timestamp: {2})". format(nice_period_start, nice_period_end, timestamp)) defer.returnValue(None) span = int(time.time()) - 3600 - timestamp if span < 900: yield logger.warning( "Waiting {0} minutes for calculating {1} - {2} (timestamp: {3})" .format(int((900 - span) / 60), nice_period_start, nice_period_end, timestamp)) defer.returnValue(None) yield logger.info( "Calculating payments for {0} - {1} (timestamp: {2}) Forced: {3}". format(nice_period_start, nice_period_end, timestamp, ignore_existing_payment_calculations)) # Calculate payments for every campaign in the round _iter = yield db_utils.get_campaign_iter() while True: campaign_doc = yield _iter.next() logger.debug(campaign_doc) if not campaign_doc: break yield logger.info("Calculating payments for campaign {0}".format( campaign_doc['campaign_id'])) # Clear campaign data and do not calculate. if campaign_doc['time_end'] < timestamp: yield logger.info("Removing old campaign: {0}".format( campaign_doc['campaign_id'])) yield stats_utils.delete_campaign(campaign_doc['campaign_id']) campaign_doc['removed'] = True yield calculate_events_payments(campaign_doc, timestamp) yield logger.info("Calculated payments for campaign {0}".format( campaign_doc['campaign_id'])) yield logger.info( "Calculated payments for {0} - {1} (timestamp: {2})".format( nice_period_start, nice_period_end, timestamp)) yield db_utils.update_payment_round(timestamp)