def get_capfill():
print("Gettings CAP stats")
prefs = get_prefs()
res = get_stats(prefs['lookback'])
out = (seq(res.data['data'])
.map(lambda row: row['Stat'])
# .filter(lambda row: int(row['clicks']) >= min_clicks)
.filter(lambda row: (
offer_exists_and_monitoring_true(row['offer_id'])))
.map(lambda row: update_in(row, ['conversions'], int))
.map(lambda row: (
assoc(row, 'conversion_cap',
get_conversion_cap(row['offer_id'],
row['affiliate_id']))))
.filter(lambda row: row['conversion_cap'] > 0)
.map(lambda row: (
assoc(row, 'value', ((row['conversions'] / prefs['lookback'])
/ row['conversion_cap']))))
.to_list())
metric = Metric.objects.get(key='cap_fill')
for row in out:
metric_log = MetricLog()
metric_log.offer_id = row['offer_id']
metric_log.affiliate_id = row['affiliate_id']
metric_log.metric = metric
metric_log.value = row['value']
metric_log.save()
celery_pubsub.publish('metric.loaded', metric_log)
def get_cr():
print("Getting CR stats")
data = get_stats()
# todo: enqueue every row to metric processor
# transform
# filter
out = (seq(data)
.map(lambda row: row['Stat'])
.filter(lambda row: (
int(row['clicks']) >= get_offer_min_clicks(row['offer_id'])))
.filter(lambda row: (
offer_exists_and_monitoring_true(row['offer_id'])))
.map(lambda row: update_in(row, ['clicks'], int))
.map(lambda row: update_in(row, ['conversions'], int))
.map(lambda row: (
assoc(row, 'value', cr(row['clicks'], row['conversions']))))
.to_list())
metric = Metric.objects.get(key='cr')
for row in out:
metric_log = MetricLog()
metric_log.offer_id = row['offer_id']
metric_log.affiliate_id = row['affiliate_id']
metric_log.metric = metric
metric_log.value = row['value']
metric_log.save()
celery_pubsub.publish('metric.loaded', metric_log)
def trigger_worker(trigger, metric_log):
print(f"trigger_worker: processing triggers {trigger}")
trigger_operator = getattr(operator, trigger.operator)
offer = Offer.objects.get(pk=metric_log.offer_id)
threshold_ = (Threshold.objects.for_trigger(trigger, metric_log,
offer.categories))
print(f"trigger_worker: using threshold {threshold_}")
if trigger_operator(metric_log.value, threshold_.value):
status = TriggerCheck.PROBLEM
else:
status = TriggerCheck.OK
# Prev trigger check
filters = {
'trigger': trigger,
'offer_id': metric_log.offer_id,
'affiliate_id': metric_log.affiliate_id
}
prev_check = (TriggerCheck.objects.filter(
**filters).order_by('-created_at').first())
# Log trigger check
values = {
'trigger': trigger,
'offer_id': metric_log.offer_id,
'affiliate_id': metric_log.affiliate_id,
'status': status
}
trigger_check = TriggerCheck(**values)
trigger_check.save()
print(f"trigger_worker: trigger check done {trigger_check}")
# Trigger Event
if prev_check:
if status != prev_check.status:
celery_pubsub.publish('trigger-event', trigger_check, metric_log,
threshold_)
print(f"trigger_worker: trigger-event published "
f"with params {trigger_check} {metric_log} {threshold_}")
else:
if status == TriggerCheck.PROBLEM:
celery_pubsub.publish('trigger-event', trigger_check, metric_log,
threshold_)
print(f"trigger_worker: trigger-event published "
f"with params {trigger_check} {metric_log} {threshold_}")
def test_subscription(self):
from celery_pubsub import publish, subscribe
res = publish('dummy', 4, 8, a15=16, a23=42).get()
self.assertListEqual(sorted(res), sorted(['e']))
subscribe('dummy', job_c)
res = publish('dummy', 4, 8, a15=16, a23=42).get()
self.assertListEqual(sorted(res), sorted(['e', 'c']))
celery_pubsub.unsubscribe('dummy', job_c)
res = publish('dummy', 4, 8, a15=16, a23=42).get()
self.assertListEqual(sorted(res), sorted(['e']))
def test_subscription(subscriber, job_c, celery_worker):
from celery_pubsub import publish, subscribe
res = publish("dummy", 4, 8, a15=16, a23=42).get()
assert sorted(res) == sorted(["e"])
subscribe("dummy", job_c)
res = publish("dummy", 4, 8, a15=16, a23=42).get()
assert sorted(res) == sorted(["e", "c"])
unsubscribe("dummy", job_c)
res = publish("dummy", 4, 8, a15=16, a23=42).get()
assert sorted(res) == sorted(["e"])
async def list_connections(request):
res = celery_pubsub.publish('connections.new.init_test',
data='something',
value=42)
res.get()
res.save(res.backend)
return json({'connections': res.as_tuple()})
def get_gr():
print("Gettings GR stats")
offer_ids = fetch_active_offers()
offer_ids = list(filter(offer_exists_and_monitoring_true, offer_ids))
offer_ids = list(filter(offer_has_goal, offer_ids))
for offer_id in offer_ids:
offer = Offer.objects.get(pk=offer_id)
# get stats for the offer
stats = get_stats(offer_id, offer.one_goal_id, 1)
out = (
seq(stats).map(lambda row: update_in(row, ['conversions'], int)).
map(lambda row: update_in(row, ['goal_id'], int)).map(
lambda row: update_in(row, ['affiliate_id'], int)).to_list())
conversions = seq(out).filter(lambda r: r['goal_id'] == 0).to_list()
goals = (seq(out).filter(
lambda r: r['goal_id'] == offer.one_goal_id).to_list())
# filter min_conversions
conversions = (seq(conversions).filter(
lambda r: r['conversions'] >= offer.min_conversions).to_list())
# populate with goal count
conversions = (seq(conversions).map(lambda r: assoc(
r, 'goals', get_goals_count(r['affiliate_id'], goals))).to_list())
# populate with gr value
conversions = (seq(conversions).map(lambda r: assoc(
r, 'value', gr(r['conversions'], r['goals']))).to_list())
# create metric
metric = Metric.objects.get(key='gtr')
for row in conversions:
metric_log = MetricLog()
metric_log.offer_id = offer_id
metric_log.affiliate_id = row['affiliate_id']
metric_log.metric = metric
metric_log.value = row['value']
metric_log.save()
celery_pubsub.publish('metric.loaded', metric_log)
def get_clicks_if_zero_conv():
print("Getting Clicks stats")
api = Hasoffers(network_token=settings.HASOFFERS_NETWORK_TOKEN,
network_id=settings.HASOFFERS_NETWORK_ID,
proxies=settings.PROXIES)
from_date = (datetime.datetime.now(pytz.timezone(settings.TIME_ZONE)) -
datetime.timedelta(days=1))
res = api.Report.getStats(fields=['Stat.clicks', 'Stat.conversions'],
groups=['Stat.offer_id', 'Stat.affiliate_id'],
filters={
'Stat.date': {
'conditional':
'GREATER_THAN_OR_EQUAL_TO',
'values': str(from_date.date())
},
'Stat.hour': {
'conditional':
'GREATER_THAN_OR_EQUAL_TO',
'values': from_date.hour
}
},
limit=10000)
out = (seq(
res.data['data']).map(lambda row: row['Stat']).filter(lambda row: (int(
row['clicks']) >= get_offer_min_clicks(row['offer_id']))).filter(
lambda row: (offer_exists_and_monitoring_true(row['offer_id']))
).map(lambda row: update_in(row, ['conversions'], int)).map(
lambda row: update_in(row, ['clicks'], int)).filter(
lambda row: row['conversions'] == 0).map(lambda row: assoc(
row, 'value', row['clicks'])).to_list())
metric = Metric.objects.get(key='clicks_zero_conv')
for row in out:
metric_log = MetricLog()
metric_log.offer_id = row['offer_id']
metric_log.affiliate_id = row['affiliate_id']
metric_log.metric = metric
metric_log.value = row['value']
metric_log.save()
celery_pubsub.publish('metric.loaded', metric_log)
def test_2(self):
res = celery_pubsub.publish('something.else', 4, 8, a15=16,
a23=42).get()
self.assertListEqual(sorted(res), sorted(['e']))
def test_1(self):
res = celery_pubsub.publish('index.low.test', 4, 8, a15=16,
a23=42).get()
self.assertListEqual(sorted(res), sorted(['d', 'e', 'f', 'g']))
def test_7(self):
from celery_pubsub import publish
res = publish('index.high.test', 4, 8, a15=16, a23=42).get()
self.assertListEqual(sorted(res), sorted(['d', 'e', 'f', 'g']))
def test_6(subscriber, celery_worker):
import celery_pubsub as pubsub
res = pubsub.publish("index.high.some.test", 4, 8, a15=16, a23=42).get()
assert sorted(res) == sorted(["d", "e", "g"])
def test_3(subscriber, celery_worker):
res = publish("index.high", 4, 8, a15=16, a23=42).get()
assert sorted(res) == sorted(["a", "d", "e"])
def test_2(subscriber, celery_worker):
res = publish("something.else", 4, 8, a15=16, a23=42).get()
assert sorted(res) == sorted(["e"])
def test_1(subscriber, celery_worker):
res = publish("index.low.test", 4, 8, a15=16, a23=42).get()
assert sorted(res) == sorted(["d", "e", "f", "g"])
def test_3(self):
res = celery_pubsub.publish('index.high', 4, 8, a15=16, a23=42).get()
self.assertListEqual(sorted(res), sorted(['a', 'd', 'e']))
def test_6(self):
import celery_pubsub as pubsub
res = pubsub.publish('index.high.some.test', 4, 8, a15=16,
a23=42).get()
self.assertListEqual(sorted(res), sorted(['d', 'e', 'g']))
def test_7(subscriber, celery_worker):
from celery_pubsub import publish
res = publish("index.high.test", 4, 8, a15=16, a23=42).get()
assert sorted(res) == sorted(["d", "e", "f", "g"])
