def load_campaigns(self):
for campaign in self.campaigns:
db_utils.update_campaign(campaign)
for banner in campaign['banners']:
banner['campaign_id'] = campaign['campaign_id']
yield db_utils.update_banner(banner)
def test_campaign_payments_more(self):
payment_percentage_cutoff = 0.5
cpv, cpc = 10, 20
timestamp = 3600
# Add campaign
cmp_doc = {"campaign_id": "campaign_id",
"time_start": 0,
"time_end": timestamp + 100,
"max_cpc": cpc,
"max_cpm": cpv,
"budget": 1000,
"filters": {'require': {},
'exclude': {}}}
yield db_utils.update_campaign(cmp_doc)
# Add 3 banners for this campaign
for i in range(3):
yield db_utils.update_banner({'banner_id': 'banner_id' + str(i),
'campaign_id': cmp_doc['campaign_id']})
# Add events for users
yield db_utils.update_event({
'campaign_id': cmp_doc['campaign_id'],
"event_id": "event2_user_id1",
"event_type": stats_consts.EVENT_TYPE_VIEW,
"timestamp": timestamp + 1,
"user_id": 'user_id1',
"banner_id": 'banner_id1',
"event_value": 0.2,
"our_keywords": {},
"their_keywords": {},
"human_score": 1.0})
yield db_utils.update_event({
'campaign_id': cmp_doc['campaign_id'],
"event_id": "event2_user_id2",
"event_type": stats_consts.EVENT_TYPE_CONVERSION,
"timestamp": timestamp + 2,
"user_id": 'user_id2',
"banner_id": 'banner_id1',
"event_value": 0.5,
"our_keywords": {},
"their_keywords": {},
"human_score": 1.0})
yield stats_legacy.calculate_events_payments(cmp_doc,
timestamp,
payment_percentage_cutoff=payment_percentage_cutoff)
# Check user values
user2_value_doc = yield db_utils.get_user_value_in_campaign(cmp_doc['campaign_id'], "user_id2")
self.assertEqual(user2_value_doc['payment'], 10)
self.assertEqual(user2_value_doc['human_score'], 1)
# User scores should be empty.
_iter = yield db_utils.get_sorted_user_score_iter(cmp_doc['campaign_id'], timestamp, limit=1)
user_score_doc = yield _iter.next()
self.assertIsNone(user_score_doc)
def create_or_update_campaign(cmpobj):
"""
Create or update campaign. Removes old banners and adds new ones.
:param cmpobj:
:return:
"""
logger = logging.getLogger(__name__)
yield logger.info("Updating campaign: {0}".format(cmpobj.campaign_id))
yield logger.debug("Updating campaign: {0}".format(cmpobj))
campaign_doc = cmpobj.to_json()
del campaign_doc['banners']
yield db_utils.update_campaign(campaign_doc)
# Delete previous banners
yield logger.info("Removing campaign banners for: {0}".format(
cmpobj.campaign_id))
yield db_utils.delete_campaign_banners(cmpobj.campaign_id)
# Update banners for campaign
yield logger.info("Updating banners for campaign: {0}".format(
cmpobj.campaign_id))
for banner in cmpobj.banners:
banner_doc = banner.to_json()
banner_doc['campaign_id'] = cmpobj.campaign_id
yield logger.debug("Updating banner: {0}".format(banner))
yield db_utils.update_banner(banner_doc)
def test_non_payable_events(self):
"""
Test for non payable events.
"""
timestamp = 3600
# Add campaign with one banner
cmp_doc = {
"campaign_id": "campaign_id",
"time_start": 0,
"time_end": 3600 + 100,
"max_cpc": 5,
"max_cpm": 5000,
"budget": 10,
"filters": {
'require': {},
'exclude': {}
}
}
yield db_utils.update_campaign(cmp_doc)
yield db_utils.update_banner({
'banner_id': 'banner_id',
'campaign_id': 'campaign_id'
})
# Add 5 events
event = {
"event_id": "event_id", # This will be modified
"event_type": "click", # This will be modified
"timestamp": timestamp + 2,
"user_id": 'test_user',
"banner_id": 'banner_id',
"campaign_id": "campaign_id",
"event_value": 5,
"our_keywords": {},
"their_keywords": {},
"human_score": 1.0
}
for i in range(5):
event['event_id'] = 'event_id_' + str(i)
event['event_type'] = str(random.randint(1000, 1001))
yield db_utils.update_event(event)
# Calculate payments
yield stats_default.calculate_events_payments(cmp_doc, timestamp)
# Check payments
_iter = yield db_utils.get_payments_iter(timestamp)
while True:
payment_doc = yield _iter.next()
if not payment_doc:
break
# Check payment reason is accepted
self.assertEqual(0, payment_doc['reason'])
# But payment is 0
self.assertEqual(0, payment_doc['payment'])
def test_budget(self):
"""
Test for hourly budget constraint. Total payments can't be higher than the campaign budget.
"""
timestamp = 3600
# Add campaign with one banner
cmp_doc = {
"campaign_id": "campaign_id",
"time_start": 0,
"time_end": 3600 + 100,
"max_cpc": 5,
"max_cpm": 5000,
"budget": 10,
"filters": {
'require': {},
'exclude': {}
}
}
yield db_utils.update_campaign(cmp_doc)
yield db_utils.update_banner({
'banner_id': 'banner_id',
'campaign_id': 'campaign_id'
})
# Add all kind of paid events
event = {
"event_id": "event_id", # This will be modified
"event_type": "click", # This will be modified
"timestamp": timestamp + 2,
"user_id": 'test_user',
"banner_id": 'banner_id',
"campaign_id": "campaign_id",
"event_value": 5,
"our_keywords": {},
"their_keywords": {},
"human_score": 1.0
}
for i, event_type in enumerate(stats_consts.PAID_EVENT_TYPES):
event['event_id'] = 'event_id_' + str(i)
event['event_type'] = event_type
yield db_utils.update_event(event)
# Calculate payments
yield stats_default.calculate_events_payments(cmp_doc, timestamp)
# Check payments
total_payments = 0
_iter = yield db_utils.get_payments_iter(timestamp)
while True:
payment_doc = yield _iter.next()
if not payment_doc:
break
total_payments += payment_doc['payment']
self.assertLessEqual(total_payments, cmp_doc['budget'])
def test_campaign_filters_add_event(self):
# Add campaign with filters
# campaign_id, time_start, time_end, max_cpc, max_cpm, budget, filters
cmp_doc = {
"campaign_id": "campaign_filter_id",
"time_start": 123,
"time_end": 234,
"max_cpc": 12,
"max_cpm": 34,
"budget": 1000,
"filters": {
'require': {
'testkey': [10]
},
'exclude': {
'testkey': ["0--5"]
}
}
}
yield db_utils.update_campaign(cmp_doc)
yield db_utils.update_banner({
'banner_id': 'banner_filter_id',
'campaign_id': 'campaign_filter_id'
})
event_data = {
'event_id': 'event_id',
'event_type': 'event_type',
'user_id': 'user_id',
'human_score': 0.5,
'publisher_id': 'publisher_id',
'timestamp': 45678,
'banner_id': "banner_filter_id",
'our_keywords': {
'testkey': 5
},
'their_keywords': {},
'event_value': None
}
pre_banner_events = yield self.get_banner_events('banner_filter_id')
# Test validation false.
response = yield self.get_response("add_events", [event_data])
self.assertIsNotNone(response)
self.assertTrue(response['result'])
banner_events = yield self.get_banner_events('banner_filter_id')
self.assertNotEqual(len(banner_events), len(pre_banner_events))
# Test validation true.
event_data['our_keywords'] = {'testkey': 10}
response = yield self.get_response("add_events", [event_data])
self.assertIsNotNone(response)
self.assertTrue(response['result'])
banner_events = yield self.get_banner_events('banner_filter_id')
self.assertEqual(len(banner_events), len(pre_banner_events) + 1)
def test_nocampaign_add_event(self):
# Event without campaign shouldn't be added
event_data = {
'event_id': 'event_id',
'event_type': 'event_type',
'user_id': 'user_id',
'human_score': 0.5,
'publisher_id': 'publisher_id',
'timestamp': 45678,
'banner_id': "banner_1",
'our_keywords': {},
'their_keywords': {},
'event_value': None
}
pre_banner_events = yield self.get_banner_events('banner_1')
response = yield self.get_response("add_events", [event_data])
self.assertIsNotNone(response)
self.assertTrue(response['result'])
post_banner_events = yield self.get_banner_events('banner_1')
self.assertNotEqual(pre_banner_events, post_banner_events)
# Add campaign for banner_1
cmp_doc = {
"campaign_id": "campaign_id",
"time_start": 123,
"time_end": 234,
"max_cpc": 100,
"max_cpm": 100,
"budget": 1000,
"filters": {}
}
yield db_utils.update_campaign(cmp_doc)
yield db_utils.update_banner({
'banner_id': 'banner_1',
'campaign_id': 'campaign_id'
})
# Test event addition with existing campaign
response = yield self.get_response("add_events", [event_data])
self.assertIsNotNone(response)
self.assertTrue(response['result'])
banner_events = yield self.get_banner_events('banner_1')
self.assertEqual(len(banner_events), len(pre_banner_events) + 1)
# Test event addition without user_id
pre_banner_events = banner_events
del event_data['user_id']
response = yield self.get_response("add_events", [event_data])
self.assertIsNotNone(response)
self.assertTrue(response['result'])
banner_events = yield self.get_banner_events('banner_1')
self.assertEqual(len(banner_events), len(pre_banner_events))
def test_adpay_task(self):
cmp_doc = {
"campaign_id": "campaign_id",
"time_start": 12345,
"time_end": 12347,
"max_cpc": 100,
"max_cpm": 200,
"budget": 1000,
"filters": {}
}
yield db_utils.update_campaign(cmp_doc)
timestamp = int(time.time()) - stats_consts.SECONDS_PER_HOUR
payment_round = yield db_utils.get_payment_round(timestamp)
payment_rounds = yield self.get_payment_rounds()
self.assertIsNone(payment_round)
self.assertEqual(len(payment_rounds), 0)
yield stats_tasks._adpay_task(timestamp)
yield stats_tasks._adpay_task(timestamp)
payment_round = yield db_utils.get_payment_round(timestamp)
payment_rounds = yield self.get_payment_rounds()
self.assertEqual(payment_round['timestamp'],
common_utils.timestamp2hour(timestamp))
self.assertEqual(len(payment_rounds), 1)
yield stats_tasks._adpay_task(timestamp, False)
cmp_doc = {
"campaign_id": "campaign_id",
"time_start": 12345,
"time_end": 12347,
"max_cpc": 100,
"max_cpm": 200,
"budget": 1000,
"filters": {}
}
yield db_utils.update_campaign(cmp_doc)
yield stats_tasks._adpay_task(0)
yield stats_tasks._adpay_task(timestamp + 10000)
def test_campaign(self):
cmp_doc = {
"campaign_id": "campaign_id",
"time_start": 12345,
"time_end": 12347,
"max_cpc": 100,
"max_cpm": 200,
"budget": 1000,
"filters": {}
}
yield db_utils.update_campaign(cmp_doc)
campaign_doc = yield db_utils.get_campaign("campaign_id")
self.assertEqual(campaign_doc['time_start'], 12345)
self.assertEqual(campaign_doc['time_end'], 12347)
self.assertEqual(campaign_doc['max_cpc'], 100)
self.assertEqual(campaign_doc['max_cpm'], 200)
self.assertEqual(campaign_doc['budget'], 1000)
yield db_utils.delete_campaign("campaign_id")
campaign_doc = yield db_utils.get_campaign("campaign_id")
self.assertEqual(None, campaign_doc)
for i in range(1000):
cmp_doc = {
"campaign_id": i,
"time_start": 12345,
"time_end": 12347,
"max_cpc": 100,
"max_cpm": 200,
"budget": 1000,
"filters": {}
}
yield db_utils.update_campaign(cmp_doc)
counter = 0
campaign_iter = yield db_utils.get_campaign_iter()
while True:
campaign_doc = yield campaign_iter.next()
if not campaign_doc:
break
counter += 1
self.assertEqual(counter, 1000)
def test_no_campaign_add_event(self):
cmp_doc = {
"campaign_id": "campaign_id",
"time_start": 123,
"time_end": 234,
"max_cpc": 100,
"max_cpm": 100,
"budget": 1000,
"filters": {}
}
yield db_utils.update_campaign(cmp_doc)
yield db_utils.update_banner({
'banner_id': 'banner_1',
'campaign_id': 'campaign_id'
})
event_data = {
'event_id': 'event_id',
'event_type': 'event_type',
'user_id': 'user_id',
'human_score': 0.5,
'publisher_id': 'publisher_id',
'timestamp': 45678,
'banner_id': "banner_1",
'our_keywords': {},
'their_keywords': {},
'event_value': None
}
campaigns = MagicMock()
campaigns.return_value = None
with patch('adpay.db.utils.get_campaign', campaigns):
response = yield self.get_response("add_events", [event_data])
self.assertIsNotNone(response)
self.assertTrue(response['result'])
# Legacy add keywords
with patch('adpay.db.utils.get_campaign', campaigns):
response = yield self.get_response("add_events", [event_data])
self.assertIsNotNone(response)
self.assertTrue(response['result'])
def delete_campaign(campaign_id):
"""
Remove campaign.
:param campaign_id:
:return:
"""
logger = logging.getLogger(__name__)
# Delete campaign banners
yield logger.info("Removing campaign banners for {0}".format(campaign_id))
yield db_utils.delete_campaign_banners(campaign_id)
# "Delete" campaign
# Set time_end to 1 - the campaign will be cleaned up during calculation task.
yield logger.info("Removing campaign for {0}".format(campaign_id))
campaign_doc = yield db_utils.get_campaign(campaign_id)
if campaign_doc:
campaign_doc['time_end'] = 1
yield db_utils.update_campaign(campaign_doc)
def test_campaign_payments(self):
payment_percentage_cutoff = 0.5
cpv, cpc = 10, 20
timestamp = 3600
cmp_doc = {"campaign_id": "campaign_id",
"time_start": 0,
"time_end": 3600 + 100,
"max_cpc": cpc,
"max_cpm": cpv,
"budget": 100000,
"filters": {'require': {},
'exclude': {}}}
yield db_utils.update_campaign(cmp_doc)
# Add 3 banners for this campaign
for i in range(3):
yield db_utils.update_banner({'banner_id': 'banner_id' + str(i),
'campaign_id': cmp_doc['campaign_id']})
# Add events for users
yield db_utils.update_event({
'campaign_id': cmp_doc['campaign_id'],
"event_id": "event1_user_id1",
"event_type": stats_consts.EVENT_TYPE_CLICK,
"timestamp": timestamp + 2,
"user_id": 'user_id1',
"banner_id": 'banner_id1',
"event_value": 0.1,
"our_keywords": {},
"their_keywords": {},
"human_score": 1})
yield stats_legacy.calculate_events_payments(cmp_doc,
timestamp,
payment_percentage_cutoff=payment_percentage_cutoff)
# Check user values
user_value_doc = yield db_utils.get_user_value_in_campaign(cmp_doc['campaign_id'], "user_id1")
self.assertEqual(user_value_doc['payment'], 20)
self.assertEqual(user_value_doc['human_score'], 1)
yield db_utils.update_event({
'campaign_id': cmp_doc['campaign_id'],
"event_id": "event2_user_id1",
"event_type": stats_consts.EVENT_TYPE_VIEW,
"timestamp": timestamp + 2,
"user_id": 'user_id1',
"banner_id": 'banner_id1',
"event_value": 0.2,
"our_keywords": {},
"their_keywords": {},
"human_score": 1})
yield db_utils.update_event({
'campaign_id': cmp_doc['campaign_id'],
"event_id": "event2_user_id2",
"event_type": stats_consts.EVENT_TYPE_CONVERSION,
"timestamp": timestamp + 2,
"user_id": 'user_id2',
"banner_id": 'banner_id1',
"event_value": 0.5,
"our_keywords": {},
"their_keywords": {},
"human_score": 1})
yield stats_legacy.calculate_events_payments(cmp_doc, 3600,
payment_percentage_cutoff=payment_percentage_cutoff)
# Check user values
user2_value_doc = yield db_utils.get_user_value_in_campaign("campaign_id", "user_id2")
self.assertEqual(user2_value_doc['payment'], 10)
self.assertEqual(user2_value_doc['human_score'], 1)
# Mock total use score
yield db_utils.update_event({
'campaign_id': cmp_doc['campaign_id'],
"event_id": "event2_user_id3",
"event_type": stats_consts.EVENT_TYPE_CONVERSION,
"timestamp": timestamp + 2,
"user_id": 'user_id3',
"banner_id": 'banner_id1',
"event_value": 0.5,
"our_keywords": {},
"their_keywords": {},
"human_score": 1.0})
with patch('adpay.stats.legacy.get_total_user_score', MagicMock(return_value=0)):
yield stats_legacy.calculate_events_payments(cmp_doc, 3600,
payment_percentage_cutoff=payment_percentage_cutoff)
# Check user values
user2_value_doc = yield db_utils.get_user_value_in_campaign("campaign_id", "user_id3")
self.assertEqual(user2_value_doc['payment'], 10)
self.assertEqual(user2_value_doc['human_score'], 1)
def test_campaign_payments_more(self):
cpv, cpc = 10, 20
timestamp = 3600
cmp_doc = {
"campaign_id": "campaign_id",
"time_start": 1234,
"time_end": 3456,
"max_cpc": cpc,
"max_cpm": cpv,
"budget": 1000,
"filters": {
'require': {},
'exclude': {}
}
}
yield db_utils.update_campaign(cmp_doc)
yield db_utils.update_banner({
'banner_id': 'banner_id1',
'campaign_id': 'campaign_id'
})
yield db_utils.update_banner({
'banner_id': 'banner_id2',
'campaign_id': 'campaign_id'
})
yield db_utils.update_banner({
'banner_id': 'banner_id3',
'campaign_id': 'campaign_id'
})
# Add events for users
yield db_utils.update_event({
"event_id": "event2_user_id1",
"event_type": stats_consts.EVENT_TYPE_VIEW,
"timestamp": 3600,
"user_id": 'user_id1',
"banner_id": 'banner_id1',
"campaign_id": "campaign_id",
"event_value": 0.2,
"our_keywords": {},
"their_keywords": {},
"human_score": 1
})
yield db_utils.update_event({
"event_id": "event2_user_id2",
"event_type": stats_consts.EVENT_TYPE_CONVERSION,
"timestamp": 3602,
"user_id": 'user_id2',
"banner_id": 'banner_id1',
"campaign_id": "campaign_id",
"event_value": 0.5,
"our_keywords": {},
"their_keywords": {},
"human_score": 1
})
yield stats_default.calculate_events_payments(cmp_doc, timestamp)
_iter = yield db_utils.get_payments_iter(timestamp)
while True:
payment_doc = yield _iter.next()
if not payment_doc:
break
if payment_doc['event_id'] == "event2_user_id1":
self.assertLess(payment_doc['payment'], 10)
if payment_doc['event_id'] == "event2_user_id2":
self.assertLess(payment_doc['payment'], 0.025)
# User scores should be empty.
_iter = yield db_utils.get_sorted_user_score_iter("campaign_id",
3600,
limit=1)
user_score_doc = yield _iter.next()
self.assertIsNone(user_score_doc)
def test_campaign_payments(self):
cpv, cpc = 10, 20
cmp_doc = {
"campaign_id": "campaign_id",
"time_start": 0,
"time_end": 3600 + 100,
"max_cpc": cpc,
"max_cpm": cpv,
"budget": 1000,
"filters": {
'require': {},
'exclude': {}
}
}
yield db_utils.update_campaign(cmp_doc)
yield db_utils.update_banner({
'banner_id': 'banner_id1',
'campaign_id': 'campaign_id'
})
# Add events for users
yield db_utils.update_event({
"event_id": "event1_user_id1",
"event_type": stats_consts.EVENT_TYPE_CLICK,
"timestamp": 3601,
"user_id": 'user_id1',
"banner_id": 'banner_id1',
"campaign_id": "campaign_id",
"event_value": 0.1,
"our_keywords": {},
"their_keywords": {},
"human_score": 1
})
timestamp = 3600
yield stats_default.calculate_events_payments(cmp_doc, timestamp)
_iter = yield db_utils.get_payments_iter(timestamp)
while True:
payment_doc = yield _iter.next()
if not payment_doc:
break
self.assertEqual(payment_doc['payment'], 20)
yield db_utils.update_event({
"event_id": "event2_user_id1",
"event_type": stats_consts.EVENT_TYPE_VIEW,
"timestamp": 3600,
"user_id": 'user_id1',
"banner_id": 'banner_id1',
"campaign_id": "campaign_id",
"event_value": 0.2,
"our_keywords": {},
"their_keywords": {},
"human_score": 1
})
yield db_utils.update_event({
"event_id": "event2_user_id2",
"event_type": stats_consts.EVENT_TYPE_CONVERSION,
"timestamp": 3602,
"user_id": 'user_id2',
"banner_id": 'banner_id1',
"campaign_id": "campaign_id",
"event_value": 0.5,
"our_keywords": {},
"their_keywords": {},
"human_score": 1
})
yield stats_default.calculate_events_payments(cmp_doc, timestamp)
_iter = yield db_utils.get_payments_iter(timestamp)
while True:
payment_doc = yield _iter.next()
if not payment_doc:
break
if payment_doc['event_id'] == "event2_user_id1":
self.assertLess(payment_doc['payment'], 10)
if payment_doc['event_id'] == "event2_user_id2":
self.assertLess(payment_doc['payment'], 0.01)