def process_message(message, non_main_ns=NON_MAIN_NS, bcast_callback=None):
msg_dict = parse_irc_message(message, non_main_ns)
def broadcast(geo_json=None):
if geo_json is not None:
geo_dict = loads(geo_json)
msg_dict['geo_ip'] = geo_dict
bcast_callback(msg_dict)
def report_failure_broadcast(error):
bcast_log.debug("could not fetch from local geoip: %s", error)
broadcast()
if msg_dict['is_anon']:
if bcast_callback:
try:
geo_url = str(LOCAL_GEOIP + '/json/' + msg_dict['user'])
except UnicodeError:
pass
else:
getPage(geo_url).addCallbacks(callback=broadcast,
errback=report_failure_broadcast)
elif bcast_callback:
broadcast()
return msg_dict
def process_message(message, non_main_ns=NON_MAIN_NS, bcast_callback=None):
msg_dict = parse_irc_message(message, non_main_ns)
def broadcast(geo_json=None):
if geo_json is not None:
geo_dict = loads(geo_json)
msg_dict['geo_ip'] = geo_dict
bcast_callback(msg_dict)
def report_failure_broadcast(error):
bcast_log.debug("could not fetch from local geoip: %s", error)
broadcast()
if msg_dict['is_anon']:
if bcast_callback:
try:
geo_url = str(LOCAL_GEOIP + '/json/' + msg_dict['user'])
except UnicodeError:
pass
else:
getPage(geo_url).addCallbacks(callback=broadcast,
errback=report_failure_broadcast)
elif bcast_callback:
broadcast()
return msg_dict
def privmsg(self, user, channel, msg):
msg = strip_colors(msg)
try:
msg = msg.decode('utf-8')
except UnicodeError as ue:
bcast_log.warn('UnicodeError: %r on IRC message %r', ue, msg)
return
msg_dict = parse_irc_message(msg, self.ns_map)
if msg_dict.get('is_anon'):
ip = msg_dict['user']
geo_loc = geolocate_anonymous_user(self.geoip_db_monitor.geoip_db,
ip)
msg_dict[self.GEO_IP_KEY] = geo_loc
self.broadcaster.broadcast(dumps(msg_dict, sort_keys=True))
def privmsg(self, user, channel, msg):
msg = strip_colors(msg)
try:
msg = msg.decode('utf-8')
except UnicodeError as ue:
bcast_log.warn('UnicodeError: %r on IRC message %r', ue, msg)
return
parsed = parse_irc_message(msg, NON_MAIN_NS)
bcast_log.info(self.geoip_db_monitor.geoip_db)
geo_loc = geolocated_anonymous_user(self.geoip_db_monitor.geoip_db,
parsed)
if geo_loc:
parsed[self.GEO_IP_KEY] = geo_loc
# Which revisions to broadcast?
self.broadcaster.broadcast(dumps(parsed, sort_keys=True))
def privmsg(self, user, channel, msg):
msg = strip_colors(msg)
try:
msg = msg.decode('utf-8')
except UnicodeError as ue:
bcast_log.warn('UnicodeError: %r on IRC message %r', ue, msg)
return
msg_dict = parse_irc_message(msg, self.ns_map)
if msg_dict.get('is_anon'):
ip = msg_dict['user']
geo_loc = geolocate_anonymous_user(self.geoip_db_monitor.geoip_db,
ip)
msg_dict[self.GEO_IP_KEY] = geo_loc
self.broadcaster.broadcast(dumps(msg_dict, sort_keys=True))
def privmsg(self, user, channel, msg):
msg = strip_colors(msg)
try:
msg = msg.decode('utf-8')
except UnicodeError as ue:
#bcast_log.warn('UnicodeError: %r on IRC message %r', ue, msg)
return
msg_dict = parse_irc_message(msg, self.ns_map)
#print "----------------------"
#print msg_dict
if 'url' in msg_dict:
#message="SomeOne editted page: %s || URL: %s"%(msg_dict['page_title'],msg_dict['url'])
#print message
u=msg_dict['url']
pt=msg_dict['page_title']
anonym=str(msg_dict['is_anon'])
username='******'
userid='192.168.1.100'
language='en'
c=str(datetime.datetime.utcnow())
#print "page_url=u,page_title=pt,user_name=username,is_anon=anonym,user_id=userid,lang=language,created=c"
db.changes.insert(page_url=u,page_title=pt,user_name=username,is_anon=anonym,user_id=userid,lang=language,created=c)
# -*- coding: utf-8 -*-
# Not really real tests, but some real/contrived IRC messages are
# handy for manual testing.
msgs = ['[[List of My Little Pony characters]] http://en.wikipedia.org/w/index.php?diff=601611011&oldid=601609204 * Anon126 * (-81) remove [[WP:SELFREF|self-reference]] in lead (this is specifically admonished in [[WP:LEADSENTENCE]]',
'[[Wikipedia:WikiProject Anime and manga/Quality articles]] http://en.wikipedia.org/w/index.php?diff=601611013&oldid=599866872 * CR4ZE * (+31) /* Good articles */ - Adding [[Big the Cat]]',
"[[Northeastern Line (Thailand)]] !N http://en.wikipedia.org/w/index.php?oldid=601611015&rcid=645601343 * Mr.BuriramCN * (+6052) [[WP:AES|←]]Created page with '{{Infobox rail line | box_width = 300px | logo = | logo_width = | logo_alt = | image = Ubolstation.jpg | imagesize = 300 | image_alt...'",
'[[User talk:2001:558:6033:77:453B:B384:FEF:E2D9]] http://en.wikipedia.org/w/index.php?diff=601611019&oldid=601610797 * Hertz1888 * (-492) Reverted 1 edit by [[Special:Contributions/67.98.243.15|67.98.243.15]]',
'[[2001:_A_Space_Odyssey_(film)]] http://en.wikipedia.org/w/index.php?diff=601611019&oldid=601610797 * Hertz1888 * (-492) Reverted 1 edit by [[Special:Contributions/67.98.243.15|67.98.243.15]]',
'[[Special:Log/abusefilter]] hit * Siddu808 * Siddu808 triggered [[Special:AbuseFilter/527|filter 527]], performing the action "createaccount" on [[Special:UserLogin]]. Actions taken: none ([[Special:AbuseLog/10210680|details]])']
from parsers import parse_irc_message
for msg in msgs:
parsed = parse_irc_message(msg)
print parsed['ns'], '-', parsed['page_title']
# -*- coding: utf-8 -*-
# Not really real tests, but some real/contrived IRC messages are
# handy for manual testing.
msgs = [
'[[List of My Little Pony characters]] http://en.wikipedia.org/w/index.php?diff=601611011&oldid=601609204 * Anon126 * (-81) remove [[WP:SELFREF|self-reference]] in lead (this is specifically admonished in [[WP:LEADSENTENCE]]',
'[[Wikipedia:WikiProject Anime and manga/Quality articles]] http://en.wikipedia.org/w/index.php?diff=601611013&oldid=599866872 * CR4ZE * (+31) /* Good articles */ - Adding [[Big the Cat]]',
"[[Northeastern Line (Thailand)]] !N http://en.wikipedia.org/w/index.php?oldid=601611015&rcid=645601343 * Mr.BuriramCN * (+6052) [[WP:AES|←]]Created page with '{{Infobox rail line | box_width = 300px | logo = | logo_width = | logo_alt = | image = Ubolstation.jpg | imagesize = 300 | image_alt...'",
'[[User talk:2001:558:6033:77:453B:B384:FEF:E2D9]] http://en.wikipedia.org/w/index.php?diff=601611019&oldid=601610797 * Hertz1888 * (-492) Reverted 1 edit by [[Special:Contributions/67.98.243.15|67.98.243.15]]',
'[[2001:_A_Space_Odyssey_(film)]] http://en.wikipedia.org/w/index.php?diff=601611019&oldid=601610797 * Hertz1888 * (-492) Reverted 1 edit by [[Special:Contributions/67.98.243.15|67.98.243.15]]',
'[[Special:Log/abusefilter]] hit * Siddu808 * Siddu808 triggered [[Special:AbuseFilter/527|filter 527]], performing the action "createaccount" on [[Special:UserLogin]]. Actions taken: none ([[Special:AbuseLog/10210680|details]])'
]
from parsers import parse_irc_message
for msg in msgs:
parsed = parse_irc_message(msg)
print parsed['ns'], '-', parsed['page_title']
def privmsg(self, user, channel, msg):
msg = strip_colors(msg)
try:
msg = msg.decode('utf-8')
except UnicodeError as ue:
bcast_log.warn('UnicodeError: %r on IRC message %r', ue, msg)
return
msg_dict = parse_irc_message(msg, self.ns_map)
print 'START!...START!...START!...START!...START!...START!...START!...START!...START!...START!...'
print msg_dict
global Edit
Edit=Edit+1
print "#####yoyoy###......"
try:
conn = mysql.connector.connect(host='localhost',
database='minor',
user='******',
password='******')
print ("string1");
if conn.is_connected():
print('Connected to MySQL database')
cursor = conn.cursor();
if 'url' in msg_dict:
tem1=msg_dict['url'];
tem2=tem1.split('.');
tem3=tem2[0].split('//')
tem4=tem3[1];
query = ("INSERT INTO edits(page_title,is_anon,url,is_unpatrolled,parent_rev_id,is_bot,is_new,is_minor,summary,flags,user_name,actions,ns,change_size,rev_id,updated,epm,lang) VALUES('%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%.5f','%s')" %(msg_dict['page_title'], msg_dict['is_anon'],msg_dict['url'],msg_dict['is_unpatrolled'],msg_dict['parent_rev_id'],msg_dict['is_bot'],msg_dict['is_new'],msg_dict['is_minor'],msg_dict['summary'],msg_dict['flags'],msg_dict['user'],msg_dict['action'],msg_dict['ns'],msg_dict['change_size'],msg_dict['rev_id'],str(datetime.datetime.now()),Edit/60.0,tem4))
# cursor = conn.cursor()
print ("string2")
cursor.execute(query)
if cursor.lastrowid:
print ("string3")
print('last insert id', cursor.lastrowid)
else:
print('last insert id not found')
conn.commit()
print ("string4")
except Error as e:
print ("string5")
print(e)
finally:
cursor.close()
conn.close()
if msg_dict.get('is_anon'):
tt=msg_dict['user']
print 'ip address:--', tt + '#########.......'
geodata=get_geodata(tt)
print "IP: %s" % geodata["ip"]
print "Country Code: %s" % geodata["countrycode"]
print "Country Name: %s" % geodata["countryname"]
print "Region Code: %s" % geodata["regioncode"]
print "Region Name: %s" % geodata["regionname"]
print "City: %s" % geodata["city"]
print "Zip Code: %s" % geodata["zipcode"]
print "Latitude: s%s" % geodata["latitude"]
print "Longitude: %s" % geodata["longitude"]
f=open("geolocation.txt","w")
f.write(geodata["longitude"]+','+geodata["latitude"]+'\n')
f.close()
lg=open("log.txt","a+")
lg.write(geodata["longitude"]+','+geodata["latitude"]+'\n')
lg.close()
# get_geodata(msg_dict[])
print "Edits per min:--%.5f" %(Edit/60.0)
print 'END!....END!....END!....END!....END!....END!....END!....END!....END!....END!....END!....'
