def __register_gears(self):
redis_conn = RedisConn().redis()
is_reg = redis_conn.get("gears_registered")
if is_reg and int(is_reg) == 1:
# Gears already registered
return
def stream_handler(item):
data = item['value']
member = json.dumps({
'device_id': data['device_id'],
'transaction_id': data['transaction_id'],
'ts': data['ts'],
})
redis.Redis().zadd(data.get('device_id'), {member: data['ts']})
Client().incrby(data['fraud_type'], 1)
GearsBuilder(reader='StreamReader',
r=redis_conn,
requirements=[
"redis", "redistimeseries"
]).foreach(stream_handler).register('data_stream')
# To avoid multiple gears from being registered for single use case, set this when register is done,
# unset this if you want to re-register the Gear when application runs again.
redis_conn.set("gears_registered", 1)
def __register_gears(self):
# Todo: need better way to check if gears is registered.
redis_conn = RedisConn().redis()
is_reg = redis_conn.get("gears_registered")
if is_reg and int(is_reg) == 1:
# Gears already registered
return
GearsBuilder(reader='StreamReader', r=redis_conn).foreach(lambda x: execute("TS.INCRBY", "clean_ts", 1))\
.register(Constants.CLEAN_STREAM_NAME)
GearsBuilder(reader='StreamReader', r=redis_conn).foreach(lambda x: execute("TS.INCRBY", "fraud_ts", 1))\
.register(Constants.FRAUD_STREAM_NAME)
redis_conn.set("gears_registered", 1)
def ad_stack(self, data):
##
# 'key' of each sorted is the device_id received.
# Ad stacked is True if the count in the range of scores (which is timestamp of event) -
# - is greater than a threshold.
#
##
ts = int(time.time() * 1000)
is_ad_stacked = False
member = json.dumps({'device_id': data['device_id'], 'transaction_id': data['transaction_id'], 'ts': ts})
RedisConn().redis().zadd(data.get('device_id'), {member: ts})
count = RedisConn().redis().zcount(data.get('device_id'), ts - Constants.AD_STACK_WINDOW, ts)
if count > Constants.AD_STACK_THRESHOLD:
is_ad_stacked = True
self.publish(data, "Fraud" if is_ad_stacked else "Clean")
return is_ad_stacked
def ip_fraud(self, data):
exists = RedisConn().bloom().cfExists(Constants.IP_CUCKOO_FILTER_NAME,
data['ip'])
if exists:
data['fraud_type'] = Constants.IP_BLACKLIST
data['status'] = Constants.FRAUD
return exists
def click_spam(self, data):
##
# 'key' of each sorted is the device_id received.
# click spam is True if the count in the range of scores (which is timestamp of event) -
# - is greater than a threshold.
#
##
is_click_spammed = False
count = RedisConn().redis().zcount(
data.get('device_id'), data['ts'] - self.click_spam_window_in_sec,
data['ts'])
if count >= self.click_spam_threshold:
is_click_spammed = True
data['fraud_type'] = Constants.CLICK_SPAM
data['status'] = Constants.FRAUD
return is_click_spammed
import os
from fastapi import Depends, FastAPI
from pymodm import connect
import routes.users as users
import routes.data_gathering as data
import routes.trainer as trainer
import routes.classifcation_project as classifcation_project
from redis_conn import RedisConn
app = FastAPI()
connect(os.environ.get('MONGO_URI') + os.environ.get("MONGO_DB_NAME"))
RedisConn.initialize(os.environ.get('REDIS_HOST'), os.environ.get('REDIS_PORT'))
app.include_router(users.router)
app.include_router(data.router)
app.include_router(trainer.router)
app.include_router(classifcation_project.router)
def publish(self, data, status): data['status'] = status stream = Constants.CLEAN_STREAM_NAME if status == 'Clean' else Constants.FRAUD_STREAM_NAME RedisConn().redis().xadd(stream, data, id='*')
def ip_fraud(self, data): exists = RedisConn().bloom().cfExists(Constants.IP_CUCKOO_FILTER_NAME, data['ip']) self.publish(data, "Clean" if not exists else "Fraud") return exists
import os
import sys
import json
import datetime
import pymodm.errors as DBerrors
from redis_conn import RedisConn
from src.image_collector import GoogleImageCollector
from pymodm import connect
from database.classification_project import ClassificationProject
from database.gathering import GatheringRun
RedisConn.initialize(os.environ.get("REDIS_HOST"),
os.environ.get("REDIS_PORT"))
connect(os.environ.get('MONGO_URI') + os.environ.get("MONGO_DB_NAME"))
data = RedisConn.CONN.blpop(
'gathering-queue') # blocking operation until data enters queue
gathering_data = json.loads(data[1]) # load json string from redis
try:
project = ClassificationProject.objects.get(
{'name': gathering_data.get('project')})
except DBerrors.DoesNotExist:
sys.exit(1)
# build a list of previously visited urls for project
visited_urls = []
for run in project.data_gathering_runs:
visited_urls = visited_urls + run.urls_visited
run = GatheringRun(search_term=gathering_data.get('search_term'))
from sanic import Sanic
from sanic.log import logger
from sanic.response import json,text
from redis_conn import RedisConn
from sanic_cors import CORS, cross_origin
redis_conn = RedisConn().redis_conn()
app = Sanic()
CORS(app, automatic_options=True)
@app.route("/api/name", methods=['GET'])
async def get_name(request):
name = redis_conn.get('name')
return json({"name": name})
@app.route("/api/name", methods=['DELETE'])
async def del_name(request):
redis_conn.delete('name')
return json({"name": "deleted"})
@app.route("/api/name", methods=['POST'])
async def new_name(request):
redis_conn.set('name',request.json['newName'])
return json({"name":request.json['newName']})
@app.route("/api/name", methods=['PUT'])
async def update_name(request):
redis_conn.set('name',request.json['updateName'])
return json({"name":request.json['updateName']})
def publish(self, data):
RedisConn().redis().xadd(Constants.STREAM_NAME, data, id='*')