def getInstance():
global twitter_api
if not twitter_api:
#auth = tweepy.OAuthHandler("xg2hLKvf1nxw1TUALvx5xA", "MkX0lDUik0mJuc6nxserddbQDWd7ZTErQN6Tf0OhOM")
auth = tweepy.OAuthHandler("m32EakXMmUaGJW4rm8wXEQ",
"TL5LSHWBNaUcW5MBG44V6piaF8JY9JPykMagtvfc")
auth.set_access_token(
"538432816-JPzoNG6xFMZ97tYthCZNKb9llfpL6rGOdNqHtZmD",
"DtqSXyezDgio8JNYvHFmlML22vsonOEWCCWhZv7I")
#auth.set_access_token("174566652-MOGbxytlmUHIN5tEMgl5rgqWdWaIQXYZ6XPyYKl1", "yem38OfoUbsoPZvOVr3k0n3X7JSUDYD8oxAKXvrJw6k")
twitter_api = API(auth_handler=auth,
host='api.twitter.com',
search_host='search.twitter.com',
cache=DBFileCache(DBCache(
timeout=-1, conn=DBSingleton.getInstance()),
FileCache("cache", timeout=-1),
timeout=-1),
secure=False,
api_root='/1',
search_root='',
retry_count=0,
retry_delay=0,
retry_errors=None,
parser=None)
return twitter_api
def loadFromDB(cls,userId, userObj = None):
cur = DBSingleton.getInstance().cursor(cursor_factory=psycopg2.extras.DictCursor)
cur.execute("SELECT * from users where id=%s;",(userId, ))
row = cur.fetchone()
if row:
return cls.loadFromDBRow(row)
return cls.loadFromTwitter(userId, userObj)
def loadFromDB(cls, userId, userObj=None):
cur = DBSingleton.getInstance().cursor(
cursor_factory=psycopg2.extras.DictCursor)
cur.execute("SELECT * from users where id=%s;", (userId, ))
row = cur.fetchone()
if row:
return cls.loadFromDBRow(row)
return cls.loadFromTwitter(userId, userObj)
def saveTFIDF(self):
if not self.TFIDFArray:
return
db = DBSingleton.getInstance()
cursor = db.cursor()
try:
cursor.execute("update users set tfidf=%s where id=%s;",
(pickle.dumps(self.getTFIDFArray()), self.id))
db.commit()
except Exception, e:
db.rollback()
print e
def saveTFIDF(self):
if not self.TFIDFArray:
return
db = DBSingleton.getInstance()
cursor = db.cursor()
try:
cursor.execute("update users set tfidf=%s where id=%s;" ,
(pickle.dumps(self.getTFIDFArray()), self.id))
db.commit()
except Exception,e:
db.rollback()
print e
def save(self):
print "SAVE"
db = DBSingleton.getInstance()
cursor = db.cursor()
try:
cursor.execute("INSERT into users(id, screen_name, obj_data, retweet_factor, impact_factor, mc_factor, tfidf) "+
"values(%s, %s, %s, %s, %s, %s, %s);",
(self.id, self.screen_name, pickle.dumps(self.userObj), self.retweetFactor, self.impactFactor, 0, pickle.dumps(self.TFIDFArray)))
db.commit()
except Exception,e:
db.rollback()
print e
def getInstance():
global twitter_api
if not twitter_api:
#auth = tweepy.OAuthHandler("xg2hLKvf1nxw1TUALvx5xA", "MkX0lDUik0mJuc6nxserddbQDWd7ZTErQN6Tf0OhOM")
auth = tweepy.OAuthHandler("m32EakXMmUaGJW4rm8wXEQ", "TL5LSHWBNaUcW5MBG44V6piaF8JY9JPykMagtvfc")
auth.set_access_token("538432816-JPzoNG6xFMZ97tYthCZNKb9llfpL6rGOdNqHtZmD", "DtqSXyezDgio8JNYvHFmlML22vsonOEWCCWhZv7I")
#auth.set_access_token("174566652-MOGbxytlmUHIN5tEMgl5rgqWdWaIQXYZ6XPyYKl1", "yem38OfoUbsoPZvOVr3k0n3X7JSUDYD8oxAKXvrJw6k")
twitter_api = API(auth_handler=auth,
host='api.twitter.com', search_host='search.twitter.com',
cache=DBFileCache(DBCache(timeout=-1, conn=DBSingleton.getInstance()), FileCache("cache", timeout=-1), timeout = -1),
secure=False, api_root='/1', search_root='',
retry_count=0, retry_delay=0, retry_errors=None,
parser=None)
return twitter_api
def save(self):
print "SAVE"
db = DBSingleton.getInstance()
cursor = db.cursor()
try:
cursor.execute(
"INSERT into users(id, screen_name, obj_data, retweet_factor, impact_factor, mc_factor, tfidf) "
+ "values(%s, %s, %s, %s, %s, %s, %s);",
(self.id, self.screen_name, pickle.dumps(
self.userObj), self.retweetFactor, self.impactFactor, 0,
pickle.dumps(self.TFIDFArray)))
db.commit()
except Exception, e:
db.rollback()
print e
def followUsers(userInfo, users):
if not len(users):
return;
api = APISingleton.getInstanceForUser(userInfo)
cur = DBSingleton.getInstance().cursor(cursor_factory=psycopg2.extras.DictCursor)
cur.execute("select nextval('tracking_seq')")
id = cur.fetchone()[0]
userId = userInfo["oauth_uid"];
cur.execute("""insert into tracking(tracking_id,tracking_user_id,tracking_date, tracking_count)
values(%s,%s,now(), %s) """, (id, userId, len(users)))
for u in users:
try:
user = u.getUserObj()
api.create_friendship(user_id=user.id)
cur.execute("""insert into tracking_users(tracking_id, current_id, user_id, user_screen_name, similarity)
values(%s,%s,%s,%s, %s)""", (id, userId, user.id, user.screen_name, u.getSim()))
except Exception,e:
print e
def followUsers(userInfo, users):
if not len(users):
return
api = APISingleton.getInstanceForUser(userInfo)
cur = DBSingleton.getInstance().cursor(
cursor_factory=psycopg2.extras.DictCursor)
cur.execute("select nextval('tracking_seq')")
id = cur.fetchone()[0]
userId = userInfo["oauth_uid"]
cur.execute(
"""insert into tracking(tracking_id,tracking_user_id,tracking_date, tracking_count)
values(%s,%s,now(), %s) """, (id, userId, len(users)))
for u in users:
try:
user = u.getUserObj()
api.create_friendship(user_id=user.id)
cur.execute(
"""insert into tracking_users(tracking_id, current_id, user_id, user_screen_name, similarity)
values(%s,%s,%s,%s, %s)""",
(id, userId, user.id, user.screen_name, u.getSim()))
except Exception, e:
print e
from tweeapi import APISingleton
from db import DBSingleton
import psycopg2.extras
def followUsers(userInfo, users):
if not len(users):
return;
api = APISingleton.getInstanceForUser(userInfo)
cur = DBSingleton.getInstance().cursor(cursor_factory=psycopg2.extras.DictCursor)
cur.execute("select nextval('tracking_seq')")
id = cur.fetchone()[0]
userId = userInfo["oauth_uid"];
cur.execute("""insert into tracking(tracking_id,tracking_user_id,tracking_date, tracking_count)
values(%s,%s,now(), %s) """, (id, userId, len(users)))
for u in users:
try:
user = u.getUserObj()
api.create_friendship(user_id=user.id)
cur.execute("""insert into tracking_users(tracking_id, current_id, user_id, user_screen_name, similarity)
values(%s,%s,%s,%s, %s)""", (id, userId, user.id, user.screen_name, u.getSim()))
except Exception,e:
print e
DBSingleton.getInstance().commit()
import hashlib
import os
sys.path.append("../lib/")
from db import DBSingleton
def getFileName(k):
md5 = hashlib.md5()
md5.update(k)
return md5.hexdigest()
limit = 1000
offset = int(sys.argv[1])
cur = DBSingleton.getInstance().cursor()
while 1:
cur.execute("select k from data limit %s offset %s", (limit, offset))
print offset
row = cur.fetchone()
while row:
fileName = getFileName(row[0])
path = os.path.join("~/cache/", fileName)
if fileName and os.path.exists(path):
os.remove(path)
os.remove(path + ".lock")
print "REMOVED ", path
row = cur.fetchone()
offset += limit
from tweeapi import APISingleton
from tweeapi.utils import EvalUser
if __name__ == "__main__":
twitterApi = APISingleton.getInstance()
tweets = twitterApi.search(q=sys.argv[1].lower(),
rpp=100,
page=1,
force=True)
tweets = [t.text for t in tweets]
tfidfArray = getTFIDFArray(tweets)
db = DBSingleton.getInstance()
cur = db.cursor(cursor_factory=psycopg2.extras.DictCursor)
cur.execute(
"select * from users where retweet_factor > 0.2 and impact_factor > 0.2 limit 1000;"
)
results = []
for row in cur:
try:
eUser = EvalUser.loadFromDBRow(row)
if eUser.getSim(tfidfArray) > 0:
results.append(eUser)
except Exception, e:
traceback.print_exc(sys.stdout)
pass
from tweeapi import APISingleton
from db import DBSingleton
import psycopg2.extras
def followUsers(userInfo, users):
if not len(users):
return
api = APISingleton.getInstanceForUser(userInfo)
cur = DBSingleton.getInstance().cursor(
cursor_factory=psycopg2.extras.DictCursor)
cur.execute("select nextval('tracking_seq')")
id = cur.fetchone()[0]
userId = userInfo["oauth_uid"]
cur.execute(
"""insert into tracking(tracking_id,tracking_user_id,tracking_date, tracking_count)
values(%s,%s,now(), %s) """, (id, userId, len(users)))
for u in users:
try:
user = u.getUserObj()
api.create_friendship(user_id=user.id)
cur.execute(
"""insert into tracking_users(tracking_id, current_id, user_id, user_screen_name, similarity)
values(%s,%s,%s,%s, %s)""",
(id, userId, user.id, user.screen_name, u.getSim()))
except Exception, e:
print e
DBSingleton.getInstance().commit()
import sys
sys.path.append("../lib/")
from db import DBSingleton
import psycopg2, psycopg2.extras
from tweeapi import APISingleton, manager
if __name__ == "__main__":
twitterApi = APISingleton.getInstance()
users = [twitterApi.get_user(sys.argv[1])]
db = DBSingleton.getInstance()
cur = DBSingleton.getInstance().cursor(cursor_factory=psycopg2.extras.DictCursor)
cur.execute("select * from site_users where oauth_uid=%s", (sys.argv[2], ))
row = cur.fetchone()
print row
manager.followUsers(row, users)
import sys,random,math,time
import hashlib
import os
sys.path.append("../lib/")
from db import DBSingleton
def getFileName(k):
md5 = hashlib.md5()
md5.update(k)
return md5.hexdigest()
limit = 1000
offset = int(sys.argv[1])
cur = DBSingleton.getInstance().cursor()
while 1:
cur.execute("select k from data limit %s offset %s", (limit, offset))
print offset
row = cur.fetchone()
while row:
fileName = getFileName(row[0])
path = os.path.join("~/cache/", fileName)
if fileName and os.path.exists(path):
os.remove(path)
os.remove(path+".lock")
print "REMOVED ",path
row = cur.fetchone()
offset += limit
results.append(eUser)
results += eUser.BFS(100, random_walk=True)
goodResults = [u for u in results if u.getImpactFactor()*u.getRetweetFactor() > 0.05 and u.getUserObj().id not in currentFriends]
print "Keyword=",keyword, "GoodResults#=",len(goodResults)
if len(goodResults) > count:
return goodResults
except Exception:
traceback.print_exc(sys.stdout)
pass
return goodResults
if __name__ == "__main__":
twitterApi = APISingleton.getInstance()
db = DBSingleton.getInstance()
cur = db.cursor(cursor_factory=psycopg2.extras.DictCursor)
cur.execute("select * from site_users where manage=1")
for row in cur:
userId = row["oauth_uid"]
keywords = row["keywords"].split(",")
for keyword in keywords:
cur2 = db.cursor(cursor_factory=psycopg2.extras.DictCursor)
cur2.execute("select user_id from tracking_users where current_id=%s", (userId, ))
currentFriends = [i["user_id"] for i in cur2]
users = getUsersToFollow(keyword.strip(), count=5, currentFriends = currentFriends)
manager.followUsers(row, users)
