def get_capfill(): print("Gettings CAP stats") prefs = get_prefs() res = get_stats(prefs['lookback']) out = (seq(res.data['data']) .map(lambda row: row['Stat']) # .filter(lambda row: int(row['clicks']) >= min_clicks) .filter(lambda row: ( offer_exists_and_monitoring_true(row['offer_id']))) .map(lambda row: update_in(row, ['conversions'], int)) .map(lambda row: ( assoc(row, 'conversion_cap', get_conversion_cap(row['offer_id'], row['affiliate_id'])))) .filter(lambda row: row['conversion_cap'] > 0) .map(lambda row: ( assoc(row, 'value', ((row['conversions'] / prefs['lookback']) / row['conversion_cap'])))) .to_list()) metric = Metric.objects.get(key='cap_fill') for row in out: metric_log = MetricLog() metric_log.offer_id = row['offer_id'] metric_log.affiliate_id = row['affiliate_id'] metric_log.metric = metric metric_log.value = row['value'] metric_log.save() celery_pubsub.publish('metric.loaded', metric_log)
def get_cr(): print("Getting CR stats") data = get_stats() # todo: enqueue every row to metric processor # transform # filter out = (seq(data) .map(lambda row: row['Stat']) .filter(lambda row: ( int(row['clicks']) >= get_offer_min_clicks(row['offer_id']))) .filter(lambda row: ( offer_exists_and_monitoring_true(row['offer_id']))) .map(lambda row: update_in(row, ['clicks'], int)) .map(lambda row: update_in(row, ['conversions'], int)) .map(lambda row: ( assoc(row, 'value', cr(row['clicks'], row['conversions'])))) .to_list()) metric = Metric.objects.get(key='cr') for row in out: metric_log = MetricLog() metric_log.offer_id = row['offer_id'] metric_log.affiliate_id = row['affiliate_id'] metric_log.metric = metric metric_log.value = row['value'] metric_log.save() celery_pubsub.publish('metric.loaded', metric_log)
def trigger_worker(trigger, metric_log): print(f"trigger_worker: processing triggers {trigger}") trigger_operator = getattr(operator, trigger.operator) offer = Offer.objects.get(pk=metric_log.offer_id) threshold_ = (Threshold.objects.for_trigger(trigger, metric_log, offer.categories)) print(f"trigger_worker: using threshold {threshold_}") if trigger_operator(metric_log.value, threshold_.value): status = TriggerCheck.PROBLEM else: status = TriggerCheck.OK # Prev trigger check filters = { 'trigger': trigger, 'offer_id': metric_log.offer_id, 'affiliate_id': metric_log.affiliate_id } prev_check = (TriggerCheck.objects.filter( **filters).order_by('-created_at').first()) # Log trigger check values = { 'trigger': trigger, 'offer_id': metric_log.offer_id, 'affiliate_id': metric_log.affiliate_id, 'status': status } trigger_check = TriggerCheck(**values) trigger_check.save() print(f"trigger_worker: trigger check done {trigger_check}") # Trigger Event if prev_check: if status != prev_check.status: celery_pubsub.publish('trigger-event', trigger_check, metric_log, threshold_) print(f"trigger_worker: trigger-event published " f"with params {trigger_check} {metric_log} {threshold_}") else: if status == TriggerCheck.PROBLEM: celery_pubsub.publish('trigger-event', trigger_check, metric_log, threshold_) print(f"trigger_worker: trigger-event published " f"with params {trigger_check} {metric_log} {threshold_}")
def test_subscription(self): from celery_pubsub import publish, subscribe res = publish('dummy', 4, 8, a15=16, a23=42).get() self.assertListEqual(sorted(res), sorted(['e'])) subscribe('dummy', job_c) res = publish('dummy', 4, 8, a15=16, a23=42).get() self.assertListEqual(sorted(res), sorted(['e', 'c'])) celery_pubsub.unsubscribe('dummy', job_c) res = publish('dummy', 4, 8, a15=16, a23=42).get() self.assertListEqual(sorted(res), sorted(['e']))
def test_subscription(subscriber, job_c, celery_worker): from celery_pubsub import publish, subscribe res = publish("dummy", 4, 8, a15=16, a23=42).get() assert sorted(res) == sorted(["e"]) subscribe("dummy", job_c) res = publish("dummy", 4, 8, a15=16, a23=42).get() assert sorted(res) == sorted(["e", "c"]) unsubscribe("dummy", job_c) res = publish("dummy", 4, 8, a15=16, a23=42).get() assert sorted(res) == sorted(["e"])
async def list_connections(request): res = celery_pubsub.publish('connections.new.init_test', data='something', value=42) res.get() res.save(res.backend) return json({'connections': res.as_tuple()})
def get_gr(): print("Gettings GR stats") offer_ids = fetch_active_offers() offer_ids = list(filter(offer_exists_and_monitoring_true, offer_ids)) offer_ids = list(filter(offer_has_goal, offer_ids)) for offer_id in offer_ids: offer = Offer.objects.get(pk=offer_id) # get stats for the offer stats = get_stats(offer_id, offer.one_goal_id, 1) out = ( seq(stats).map(lambda row: update_in(row, ['conversions'], int)). map(lambda row: update_in(row, ['goal_id'], int)).map( lambda row: update_in(row, ['affiliate_id'], int)).to_list()) conversions = seq(out).filter(lambda r: r['goal_id'] == 0).to_list() goals = (seq(out).filter( lambda r: r['goal_id'] == offer.one_goal_id).to_list()) # filter min_conversions conversions = (seq(conversions).filter( lambda r: r['conversions'] >= offer.min_conversions).to_list()) # populate with goal count conversions = (seq(conversions).map(lambda r: assoc( r, 'goals', get_goals_count(r['affiliate_id'], goals))).to_list()) # populate with gr value conversions = (seq(conversions).map(lambda r: assoc( r, 'value', gr(r['conversions'], r['goals']))).to_list()) # create metric metric = Metric.objects.get(key='gtr') for row in conversions: metric_log = MetricLog() metric_log.offer_id = offer_id metric_log.affiliate_id = row['affiliate_id'] metric_log.metric = metric metric_log.value = row['value'] metric_log.save() celery_pubsub.publish('metric.loaded', metric_log)
def get_clicks_if_zero_conv(): print("Getting Clicks stats") api = Hasoffers(network_token=settings.HASOFFERS_NETWORK_TOKEN, network_id=settings.HASOFFERS_NETWORK_ID, proxies=settings.PROXIES) from_date = (datetime.datetime.now(pytz.timezone(settings.TIME_ZONE)) - datetime.timedelta(days=1)) res = api.Report.getStats(fields=['Stat.clicks', 'Stat.conversions'], groups=['Stat.offer_id', 'Stat.affiliate_id'], filters={ 'Stat.date': { 'conditional': 'GREATER_THAN_OR_EQUAL_TO', 'values': str(from_date.date()) }, 'Stat.hour': { 'conditional': 'GREATER_THAN_OR_EQUAL_TO', 'values': from_date.hour } }, limit=10000) out = (seq( res.data['data']).map(lambda row: row['Stat']).filter(lambda row: (int( row['clicks']) >= get_offer_min_clicks(row['offer_id']))).filter( lambda row: (offer_exists_and_monitoring_true(row['offer_id'])) ).map(lambda row: update_in(row, ['conversions'], int)).map( lambda row: update_in(row, ['clicks'], int)).filter( lambda row: row['conversions'] == 0).map(lambda row: assoc( row, 'value', row['clicks'])).to_list()) metric = Metric.objects.get(key='clicks_zero_conv') for row in out: metric_log = MetricLog() metric_log.offer_id = row['offer_id'] metric_log.affiliate_id = row['affiliate_id'] metric_log.metric = metric metric_log.value = row['value'] metric_log.save() celery_pubsub.publish('metric.loaded', metric_log)
def test_2(self): res = celery_pubsub.publish('something.else', 4, 8, a15=16, a23=42).get() self.assertListEqual(sorted(res), sorted(['e']))
def test_1(self): res = celery_pubsub.publish('index.low.test', 4, 8, a15=16, a23=42).get() self.assertListEqual(sorted(res), sorted(['d', 'e', 'f', 'g']))
def test_7(self): from celery_pubsub import publish res = publish('index.high.test', 4, 8, a15=16, a23=42).get() self.assertListEqual(sorted(res), sorted(['d', 'e', 'f', 'g']))
def test_6(subscriber, celery_worker): import celery_pubsub as pubsub res = pubsub.publish("index.high.some.test", 4, 8, a15=16, a23=42).get() assert sorted(res) == sorted(["d", "e", "g"])
def test_3(subscriber, celery_worker): res = publish("index.high", 4, 8, a15=16, a23=42).get() assert sorted(res) == sorted(["a", "d", "e"])
def test_2(subscriber, celery_worker): res = publish("something.else", 4, 8, a15=16, a23=42).get() assert sorted(res) == sorted(["e"])
def test_1(subscriber, celery_worker): res = publish("index.low.test", 4, 8, a15=16, a23=42).get() assert sorted(res) == sorted(["d", "e", "f", "g"])
def test_3(self): res = celery_pubsub.publish('index.high', 4, 8, a15=16, a23=42).get() self.assertListEqual(sorted(res), sorted(['a', 'd', 'e']))
def test_6(self): import celery_pubsub as pubsub res = pubsub.publish('index.high.some.test', 4, 8, a15=16, a23=42).get() self.assertListEqual(sorted(res), sorted(['d', 'e', 'g']))
def test_7(subscriber, celery_worker): from celery_pubsub import publish res = publish("index.high.test", 4, 8, a15=16, a23=42).get() assert sorted(res) == sorted(["d", "e", "f", "g"])