def save_tweet(data): global tracker print "save_tweet:", __name__ #print data indata = data s1 = Subject() s1.set_platform("web") s1.set_user_id(str(indata.get("user_id"))) s1.set_lang(str(indata.get("lang"))) #s1.set_ip_address(str(indata.get("i_ip"))) s1.set_useragent(str(indata.get("source"))) tracker.set_subject(s1) tracker.track_self_describing_event( SelfDescribingJson( "iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0", { "data": { "data": indata }, "schema": "iglu:com.rbox24/" + args.sp_app_id + "/jsonschema/1-0-0" })) tracker.flush() print "Tweet sent to collector, time:", time.time()
def track(type, data): """ Called for the tracking of events input: type of event to track, data to pass output: No return """ emitter = Emitter(SNOWPLOW_MICRO_URI, buffer_size=1, on_success=success, on_failure=failure) tracker = Tracker(emitter) # Dictionary to contain all events we want to track and their corresponding methods. # So we can have a more generic way to track dict = { PAGE_VIEW: tracker.track_page_view, FORM_SUBMIT: tracker.track_form_submit, STRUCT: tracker.track_struct_event } subject = Subject() subject.set_platform("pc") tracker.set_subject(subject) if isinstance(data, list): dict[type](*data) else: dict[type](data) logger.setLevel(10)
def listen(): print "attach" # Kafka consumer = KafkaConsumer(bootstrap_servers=os.environ["KAFKA_BOOTSTRAP_SRVS"], group_id=os.environ["KAFKA_GROUP_ID"]) consumer.subscribe([os.environ["KAFKA_SOURCE_TOPIC"]]) # Snowplow e = Emitter(os.environ["SP_COLLECTOR_URI"],protocol=os.environ["SP_COLLECTOR_PROTOCOL"],port=int(os.environ["SP_COLLECTOR_PORT"]),method=os.environ["SP_COLLECTOR_METHOD"]) t = Tracker(emitters=e,namespace="cf",app_id=str(os.environ["APP_ID"]),encode_base64=True) for msg in consumer: # try: indata = json.loads(msg.value) s1 = Subject() s1.set_platform("app") s1.set_user_id("??") s1.set_lang("??") s1.set_ip_address("0.0.0.0") s1.set_useragent("??") t.set_subject(s1) t.track_self_describing_event(SelfDescribingJson("iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0",{ "data":{ "data": indata }, "schema": "iglu:"+os.environ["OPERATOR_ID"]+"/"+os.environ["APP_ID"]+"/jsonschema/1-0-0" })) t.flush() except Exception,Argument: print "Error:",str(Argument)
def save_tweet(data): #print "save_tweet" #print data indata = data e = Emitter(args.sp_collector_uri, protocol=args.sp_collector_protocol, port=int(args.sp_collector_port), method=args.sp_collector_method) t = Tracker(emitters=e, namespace="cf", app_id=args.sp_app_id, encode_base64=True) s1 = Subject() s1.set_platform("web") s1.set_user_id(str(indata.get("user_id"))) s1.set_lang(str(indata.get("lang"))) #s1.set_ip_address(str(indata.get("i_ip"))) s1.set_useragent(str(indata.get("source"))) t.set_subject(s1) t.track_self_describing_event( SelfDescribingJson( "iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0", { "data": { "data": indata }, "schema": "iglu:com.rbox24/" + args.sp_app_id + "/jsonschema/1-0-0" })) t.flush() print "Tweet sent to collector, time:", time.time()
import pytz import datetime from collections import Counter from snowplow_tracker import Subject, Tracker, Emitter, SelfDescribingJson from urllib.request import urlopen # get config file and read with open("config.yaml", 'r') as configfile: cfg = yaml.load(configfile) # initialze Snowplow Tracker e = Emitter(cfg['snpl-mini']['server'], protocol="http", method="post") t = Tracker(e, app_id="weather_info") s = Subject() s.set_timezone("Australia/Sydney") s.set_platform("srv") # Return most commin in list def most_common(lst): data = Counter(lst) return data.most_common(1)[0][0] # finding key and value searching in dic def gen_dict_extract(node, kv): if isinstance(node, list): for i in node: for x in gen_dict_extract(i, kv): yield x elif isinstance(node, dict): if kv in node: