def get_banner_events_iter(banner_id, timestamp):
"""
Fetch all banner documents from the database.
:param banner_id: Banner identifier.
:param timestamp: Time in seconds since the epoch, used for getting the full hour timestamp.
:return: Iterable events collection (QueryIterator)
"""
timestamp = common_utils.timestamp2hour(timestamp)
collection = yield db.get_event_collection()
defer.returnValue(QueryIterator(collection.find({
'banner_id': banner_id,
'timestamp': common_utils.timestamp2hour(timestamp)
}, cursor=True)))
def update_user_score(campaign_id, timestamp, user_id, score):
"""
Update user score with new score and timestamp, per campaign.
:param campaign_id:
:param timestamp:
:param user_id:
:param score:
:return:
"""
timestamp = common_utils.timestamp2hour(timestamp)
collection = yield db.get_user_score_collection()
return_value = yield collection.replace_one({
'campaign_id': campaign_id,
'timestamp': timestamp,
'user_id': user_id
}, {
'campaign_id': campaign_id,
'timestamp': timestamp,
'user_id': user_id,
'score': score
}, upsert=True)
defer.returnValue(return_value)
def step_impl(context, timestamp):
def test_doc(doc):
assert doc is not None
assert 'timestamp' in doc
assert doc['timestamp'] <= timestamp
timestamp = timestamp2hour(timestamp)
last_round_doc = utils.get_payment_round(timestamp)
last_round_doc.addCallback(test_doc)
def delete_payment_round(timestamp):
"""
Remove the payment round from collection.
:param timestamp:
:return:
"""
timestamp = common_utils.timestamp2hour(timestamp)
collection = yield db.get_payment_rounds_collection()
return_value = yield collection.delete_many({'timestamp': timestamp})
defer.returnValue(return_value)
def delete_user_scores(campaign_id, timestamp):
"""
:param campaign_id:
:param timestamp:
:return:
"""
timestamp = common_utils.timestamp2hour(timestamp)
collection = yield db.get_user_score_collection()
return_value = yield collection.delete_many({'campaign_id': campaign_id, 'timestamp': timestamp})
defer.returnValue(return_value)
def get_payment_round(timestamp):
"""
Fetch a payment round document from the database.
:param timestamp: Timestamp (epoch, seconds, full hour) for this request.
:return: Payment round document.
"""
timestamp = common_utils.timestamp2hour(timestamp)
collection = yield db.get_payment_rounds_collection()
return_value = yield collection.find_one({'timestamp': timestamp})
defer.returnValue(return_value)
def get_payments_iter(timestamp):
"""
Fetch all payment documents from the database.
:param timestamp: Timestamp (epoch, seconds, full hour) for this request.
:return: Iterable payment information.
"""
timestamp = common_utils.timestamp2hour(timestamp)
collection = yield db.get_payment_collection()
defer.returnValue(QueryIterator(collection.find({
'timestamp': timestamp
}, cursor=True)))
def update_payment_round(timestamp):
"""
Create or update a payment round.
:param timestamp:
:return:
"""
timestamp = common_utils.timestamp2hour(timestamp)
collection = yield db.get_payment_rounds_collection()
round_doc = {'timestamp': timestamp}
return_value = yield collection.update(round_doc, round_doc, upsert=True)
defer.returnValue(return_value)
def get_sorted_user_score_iter(campaign_id, timestamp, limit):
"""
:param campaign_id:
:param timestamp:
:param limit:
:return: Descending by score sorted list of user score to limit.
"""
timestamp = common_utils.timestamp2hour(timestamp)
collection = yield db.get_user_score_collection()
defer.returnValue(QueryIterator(collection.find({'campaign_id': campaign_id, 'timestamp': timestamp},
sort=txfilter.sort(txfilter.DESCENDING("score")),
limit=limit, cursor=True)))
def update_event(event_doc):
"""
Create or update an event if it doesn't exist.
:param event_doc: Event document
:return:
"""
event_doc['timestamp'] = common_utils.timestamp2hour(event_doc['timestamp'])
collection = yield db.get_event_collection()
return_value = yield collection.replace_one({'event_id': event_doc['event_id']},
event_doc,
upsert=True)
defer.returnValue(return_value)
def get_distinct_users_from_events(campaign_id, timestamp):
"""
Fetch distinct user identifiers for this campaign, for this timestamp.
:param campaign_id: Campaign identifier.
:param timestamp: Time in seconds since the epoch, used for getting the full hour timestamp.
:return: List of distinct users ids.
"""
timestamp = common_utils.timestamp2hour(timestamp)
collection = yield db.get_event_collection()
return_values = yield collection.distinct(key='user_id',
filter={'timestamp': timestamp,
'campaign_id': campaign_id})
defer.returnValue(return_values)
def get_events_per_user_iter(campaign_id, timestamp, uid):
"""
Fetch all events for this campaign, for this timestamp, for this uid.
:param campaign_id: Campaign identifier.
:param timestamp: Time in seconds since the epoch, used for getting the full hour timestamp.
:param uid: User identifier.
:return: Iterable events for the user.
"""
timestamp = common_utils.timestamp2hour(timestamp)
collection = yield db.get_event_collection()
defer.returnValue(QueryIterator(collection.find({
'user_id': uid,
'campaign_id': campaign_id,
'timestamp': timestamp
}, cursor=True)))
def test_adpay_task(self):
cmp_doc = {
"campaign_id": "campaign_id",
"time_start": 12345,
"time_end": 12347,
"max_cpc": 100,
"max_cpm": 200,
"budget": 1000,
"filters": {}
}
yield db_utils.update_campaign(cmp_doc)
timestamp = int(time.time()) - stats_consts.SECONDS_PER_HOUR
payment_round = yield db_utils.get_payment_round(timestamp)
payment_rounds = yield self.get_payment_rounds()
self.assertIsNone(payment_round)
self.assertEqual(len(payment_rounds), 0)
yield stats_tasks._adpay_task(timestamp)
yield stats_tasks._adpay_task(timestamp)
payment_round = yield db_utils.get_payment_round(timestamp)
payment_rounds = yield self.get_payment_rounds()
self.assertEqual(payment_round['timestamp'],
common_utils.timestamp2hour(timestamp))
self.assertEqual(len(payment_rounds), 1)
yield stats_tasks._adpay_task(timestamp, False)
cmp_doc = {
"campaign_id": "campaign_id",
"time_start": 12345,
"time_end": 12347,
"max_cpc": 100,
"max_cpm": 200,
"budget": 1000,
"filters": {}
}
yield db_utils.update_campaign(cmp_doc)
yield stats_tasks._adpay_task(0)
yield stats_tasks._adpay_task(timestamp + 10000)
def step_impl(context, number, timestamp, event_id):
class QueryAnalyzer:
def __init__(self, dfr):
self.finished = False
self.length = 0
dfr.addCallback(self.test_query)
def test_query(self, query):
assert query is not None
assert isinstance(query, utils.QueryIterator)
while not self.finished:
payment_doc = yield query.next()
if not payment_doc:
self.finished = True
self.length += 1
timestamp = timestamp2hour(timestamp)
qa = QueryAnalyzer(utils.get_payments_iter(timestamp))
assert qa.length == int(number)
def update_event_payment(campaign_id, timestamp, event_id, payment, reason):
"""
Create or update payment information for event.
:param campaign_id: Campaign identifier.
:param timestamp: Timestamp (epoch, in seconds, full hour)
:param event_id: Event identifier
:param payment: Payment amount.
:param reason: Reason (payment classifier).
:return:
"""
timestamp = common_utils.timestamp2hour(timestamp)
collection = yield db.get_payment_collection()
return_value = yield collection.replace_one({'event_id': event_id, 'campaign_id': campaign_id}, {
'timestamp': timestamp,
'event_id': event_id,
'payment': payment,
'campaign_id': campaign_id,
'reason': reason
}, upsert=True)
defer.returnValue(return_value)
def test_timestamp2hour(self):
ts = common_utils.timestamp2hour(time.time())
self.assertIs(type(ts), int)
def _adpay_task(timestamp=None, ignore_existing_payment_calculations=False):
"""
Task calculate payments and update user profiles only once a hour.
:param timestamp: Timestamp for which to calculate the payments.
:param ignore_existing_payment_calculations: Check first if the payment is already calculated.
"""
logger = logging.getLogger(__name__)
# As recalculate only finished hours, take timestamp from an hour before now.
if timestamp is None:
yield logger.warning(
"No timestamp found for recalculation, using current time.")
timestamp = int(time.time()) - 3600
timestamp = common_utils.timestamp2hour(timestamp)
nice_period_start = datetime.fromtimestamp(timestamp)
nice_period_end = datetime.fromtimestamp(timestamp) + timedelta(
seconds=3600)
if not ignore_existing_payment_calculations:
last_round_doc = yield db_utils.get_payment_round(timestamp)
if last_round_doc is not None:
yield logger.warning(
"Payment already calculated for {0} - {1} (timestamp: {2})".
format(nice_period_start, nice_period_end, timestamp))
defer.returnValue(None)
span = int(time.time()) - 3600 - timestamp
if span < 900:
yield logger.warning(
"Waiting {0} minutes for calculating {1} - {2} (timestamp: {3})"
.format(int((900 - span) / 60), nice_period_start,
nice_period_end, timestamp))
defer.returnValue(None)
yield logger.info(
"Calculating payments for {0} - {1} (timestamp: {2}) Forced: {3}".
format(nice_period_start, nice_period_end, timestamp,
ignore_existing_payment_calculations))
# Calculate payments for every campaign in the round
_iter = yield db_utils.get_campaign_iter()
while True:
campaign_doc = yield _iter.next()
logger.debug(campaign_doc)
if not campaign_doc:
break
yield logger.info("Calculating payments for campaign {0}".format(
campaign_doc['campaign_id']))
# Clear campaign data and do not calculate.
if campaign_doc['time_end'] < timestamp:
yield logger.info("Removing old campaign: {0}".format(
campaign_doc['campaign_id']))
yield stats_utils.delete_campaign(campaign_doc['campaign_id'])
campaign_doc['removed'] = True
yield calculate_events_payments(campaign_doc, timestamp)
yield logger.info("Calculated payments for campaign {0}".format(
campaign_doc['campaign_id']))
yield logger.info(
"Calculated payments for {0} - {1} (timestamp: {2})".format(
nice_period_start, nice_period_end, timestamp))
yield db_utils.update_payment_round(timestamp)
