def process_message(message, non_main_ns=NON_MAIN_NS, bcast_callback=None): msg_dict = parse_irc_message(message, non_main_ns) def broadcast(geo_json=None): if geo_json is not None: geo_dict = loads(geo_json) msg_dict['geo_ip'] = geo_dict bcast_callback(msg_dict) def report_failure_broadcast(error): bcast_log.debug("could not fetch from local geoip: %s", error) broadcast() if msg_dict['is_anon']: if bcast_callback: try: geo_url = str(LOCAL_GEOIP + '/json/' + msg_dict['user']) except UnicodeError: pass else: getPage(geo_url).addCallbacks(callback=broadcast, errback=report_failure_broadcast) elif bcast_callback: broadcast() return msg_dict
def privmsg(self, user, channel, msg): msg = strip_colors(msg) try: msg = msg.decode('utf-8') except UnicodeError as ue: bcast_log.warn('UnicodeError: %r on IRC message %r', ue, msg) return msg_dict = parse_irc_message(msg, self.ns_map) if msg_dict.get('is_anon'): ip = msg_dict['user'] geo_loc = geolocate_anonymous_user(self.geoip_db_monitor.geoip_db, ip) msg_dict[self.GEO_IP_KEY] = geo_loc self.broadcaster.broadcast(dumps(msg_dict, sort_keys=True))
def privmsg(self, user, channel, msg): msg = strip_colors(msg) try: msg = msg.decode('utf-8') except UnicodeError as ue: bcast_log.warn('UnicodeError: %r on IRC message %r', ue, msg) return parsed = parse_irc_message(msg, NON_MAIN_NS) bcast_log.info(self.geoip_db_monitor.geoip_db) geo_loc = geolocated_anonymous_user(self.geoip_db_monitor.geoip_db, parsed) if geo_loc: parsed[self.GEO_IP_KEY] = geo_loc # Which revisions to broadcast? self.broadcaster.broadcast(dumps(parsed, sort_keys=True))
def privmsg(self, user, channel, msg): msg = strip_colors(msg) try: msg = msg.decode('utf-8') except UnicodeError as ue: #bcast_log.warn('UnicodeError: %r on IRC message %r', ue, msg) return msg_dict = parse_irc_message(msg, self.ns_map) #print "----------------------" #print msg_dict if 'url' in msg_dict: #message="SomeOne editted page: %s || URL: %s"%(msg_dict['page_title'],msg_dict['url']) #print message u=msg_dict['url'] pt=msg_dict['page_title'] anonym=str(msg_dict['is_anon']) username='******' userid='192.168.1.100' language='en' c=str(datetime.datetime.utcnow()) #print "page_url=u,page_title=pt,user_name=username,is_anon=anonym,user_id=userid,lang=language,created=c" db.changes.insert(page_url=u,page_title=pt,user_name=username,is_anon=anonym,user_id=userid,lang=language,created=c)
# -*- coding: utf-8 -*- # Not really real tests, but some real/contrived IRC messages are # handy for manual testing. msgs = ['[[List of My Little Pony characters]] http://en.wikipedia.org/w/index.php?diff=601611011&oldid=601609204 * Anon126 * (-81) remove [[WP:SELFREF|self-reference]] in lead (this is specifically admonished in [[WP:LEADSENTENCE]]', '[[Wikipedia:WikiProject Anime and manga/Quality articles]] http://en.wikipedia.org/w/index.php?diff=601611013&oldid=599866872 * CR4ZE * (+31) /* Good articles */ - Adding [[Big the Cat]]', "[[Northeastern Line (Thailand)]] !N http://en.wikipedia.org/w/index.php?oldid=601611015&rcid=645601343 * Mr.BuriramCN * (+6052) [[WP:AES|←]]Created page with '{{Infobox rail line | box_width = 300px | logo = | logo_width = | logo_alt = | image = Ubolstation.jpg | imagesize = 300 | image_alt...'", '[[User talk:2001:558:6033:77:453B:B384:FEF:E2D9]] http://en.wikipedia.org/w/index.php?diff=601611019&oldid=601610797 * Hertz1888 * (-492) Reverted 1 edit by [[Special:Contributions/67.98.243.15|67.98.243.15]]', '[[2001:_A_Space_Odyssey_(film)]] http://en.wikipedia.org/w/index.php?diff=601611019&oldid=601610797 * Hertz1888 * (-492) Reverted 1 edit by [[Special:Contributions/67.98.243.15|67.98.243.15]]', '[[Special:Log/abusefilter]] hit * Siddu808 * Siddu808 triggered [[Special:AbuseFilter/527|filter 527]], performing the action "createaccount" on [[Special:UserLogin]]. Actions taken: none ([[Special:AbuseLog/10210680|details]])'] from parsers import parse_irc_message for msg in msgs: parsed = parse_irc_message(msg) print parsed['ns'], '-', parsed['page_title']
# -*- coding: utf-8 -*- # Not really real tests, but some real/contrived IRC messages are # handy for manual testing. msgs = [ '[[List of My Little Pony characters]] http://en.wikipedia.org/w/index.php?diff=601611011&oldid=601609204 * Anon126 * (-81) remove [[WP:SELFREF|self-reference]] in lead (this is specifically admonished in [[WP:LEADSENTENCE]]', '[[Wikipedia:WikiProject Anime and manga/Quality articles]] http://en.wikipedia.org/w/index.php?diff=601611013&oldid=599866872 * CR4ZE * (+31) /* Good articles */ - Adding [[Big the Cat]]', "[[Northeastern Line (Thailand)]] !N http://en.wikipedia.org/w/index.php?oldid=601611015&rcid=645601343 * Mr.BuriramCN * (+6052) [[WP:AES|←]]Created page with '{{Infobox rail line | box_width = 300px | logo = | logo_width = | logo_alt = | image = Ubolstation.jpg | imagesize = 300 | image_alt...'", '[[User talk:2001:558:6033:77:453B:B384:FEF:E2D9]] http://en.wikipedia.org/w/index.php?diff=601611019&oldid=601610797 * Hertz1888 * (-492) Reverted 1 edit by [[Special:Contributions/67.98.243.15|67.98.243.15]]', '[[2001:_A_Space_Odyssey_(film)]] http://en.wikipedia.org/w/index.php?diff=601611019&oldid=601610797 * Hertz1888 * (-492) Reverted 1 edit by [[Special:Contributions/67.98.243.15|67.98.243.15]]', '[[Special:Log/abusefilter]] hit * Siddu808 * Siddu808 triggered [[Special:AbuseFilter/527|filter 527]], performing the action "createaccount" on [[Special:UserLogin]]. Actions taken: none ([[Special:AbuseLog/10210680|details]])' ] from parsers import parse_irc_message for msg in msgs: parsed = parse_irc_message(msg) print parsed['ns'], '-', parsed['page_title']
def privmsg(self, user, channel, msg): msg = strip_colors(msg) try: msg = msg.decode('utf-8') except UnicodeError as ue: bcast_log.warn('UnicodeError: %r on IRC message %r', ue, msg) return msg_dict = parse_irc_message(msg, self.ns_map) print 'START!...START!...START!...START!...START!...START!...START!...START!...START!...START!...' print msg_dict global Edit Edit=Edit+1 print "#####yoyoy###......" try: conn = mysql.connector.connect(host='localhost', database='minor', user='******', password='******') print ("string1"); if conn.is_connected(): print('Connected to MySQL database') cursor = conn.cursor(); if 'url' in msg_dict: tem1=msg_dict['url']; tem2=tem1.split('.'); tem3=tem2[0].split('//') tem4=tem3[1]; query = ("INSERT INTO edits(page_title,is_anon,url,is_unpatrolled,parent_rev_id,is_bot,is_new,is_minor,summary,flags,user_name,actions,ns,change_size,rev_id,updated,epm,lang) VALUES('%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%.5f','%s')" %(msg_dict['page_title'], msg_dict['is_anon'],msg_dict['url'],msg_dict['is_unpatrolled'],msg_dict['parent_rev_id'],msg_dict['is_bot'],msg_dict['is_new'],msg_dict['is_minor'],msg_dict['summary'],msg_dict['flags'],msg_dict['user'],msg_dict['action'],msg_dict['ns'],msg_dict['change_size'],msg_dict['rev_id'],str(datetime.datetime.now()),Edit/60.0,tem4)) # cursor = conn.cursor() print ("string2") cursor.execute(query) if cursor.lastrowid: print ("string3") print('last insert id', cursor.lastrowid) else: print('last insert id not found') conn.commit() print ("string4") except Error as e: print ("string5") print(e) finally: cursor.close() conn.close() if msg_dict.get('is_anon'): tt=msg_dict['user'] print 'ip address:--', tt + '#########.......' geodata=get_geodata(tt) print "IP: %s" % geodata["ip"] print "Country Code: %s" % geodata["countrycode"] print "Country Name: %s" % geodata["countryname"] print "Region Code: %s" % geodata["regioncode"] print "Region Name: %s" % geodata["regionname"] print "City: %s" % geodata["city"] print "Zip Code: %s" % geodata["zipcode"] print "Latitude: s%s" % geodata["latitude"] print "Longitude: %s" % geodata["longitude"] f=open("geolocation.txt","w") f.write(geodata["longitude"]+','+geodata["latitude"]+'\n') f.close() lg=open("log.txt","a+") lg.write(geodata["longitude"]+','+geodata["latitude"]+'\n') lg.close() # get_geodata(msg_dict[]) print "Edits per min:--%.5f" %(Edit/60.0) print 'END!....END!....END!....END!....END!....END!....END!....END!....END!....END!....END!....'