def getCommTags(self, uid):
redis = RedisCluster.getRedis(uid, NEIGH_TAGS_DB)
cache = redis.hgetall(uid)
ret = {}
for k, v in cache:
ret[k] = v.split(",")
return ret
def getCommTags(self, uid):
redis = RedisCluster.getRedis(uid, NEIGH_TAGS_DB)
cache = redis.hgetall(uid)
ret = {}
for k, v in cache:
ret[k] = v.split(",")
return ret
def __init__(self, config):
logging.info("initialize data access layer")
self.config = config
snservers=[server.strip() for server in config.get('sncache','hosts').split(",")]
snports=[int(port) for port in config.get('sncache', 'ports').split(',')]
self.snCluster=RedisCluster(zip(snservers, snports))
self.snCluster.start()
uservers=config.get('profilecache','hosts').split(",")
uports=[int(port) for port in config.get('profilecache', 'ports').split(',')]
self.profileCluster=RedisCluster(zip(uservers, uports))
self.profileCluster.start()
jobservers=[server.strip() for server in config.get('jobqueue','hosts').split(",")]
jobports=[int(port) for port in config.get('jobqueue', 'ports').split(',')]
self.jobCluster=RedisCluster(zip(jobservers, jobports))
self.jobCluster.start()
return set()
else:
return self.neighbours(nodeID)
"""
read neighbours of nodeID from redis cluster
"""
def fetchNode(self, nodeID):
neighbours = self.snDao.getUserFriendsID(nodeID)
if len(neighbours) > 0:
self.addEdges(nodeID, neighbours)
if __name__ == "__main__":
# csvReader = csv.reader(file(sys.argv[1],'rb'), csv.excel_tab)
# i = 0
# g = Graph()
# for line in csvReader:
# edgeArr = line
# g.addEdge(i, edgeArr[0], edgeArr[1], 1.0)
# i+=1
# g.printGraph()
dataCluster = RedisCluster([ ("10.11.1.51", 6379),
("10.11.1.52", 6379), ("10.11.1.53", 6379), ("10.11.1.54", 6379), ("10.11.1.55", 6379),
("10.11.1.56", 6379), ("10.11.1.57", 6379), ("10.11.1.58", 6379), ("10.11.1.61", 6379),
("10.11.1.46", 6379), ("10.11.1.63", 6379)])
dataCluster.start()
#graphCache = GraphCache(dataCluster)
#print str(graphCache.egoNetwork("1000048833"))
#dataCluster.getRedis("1897953162", SN_DB).delete("1897953162")
dataCluster.getRedis("1650507560", SN_DB).delete("1650507560")
class DataLayer(object):
def __init__(self, config):
logging.info("initialize data access layer")
self.config = config
snservers=[server.strip() for server in config.get('sncache','hosts').split(",")]
snports=[int(port) for port in config.get('sncache', 'ports').split(',')]
self.snCluster=RedisCluster(zip(snservers, snports))
self.snCluster.start()
uservers=config.get('profilecache','hosts').split(",")
uports=[int(port) for port in config.get('profilecache', 'ports').split(',')]
self.profileCluster=RedisCluster(zip(uservers, uports))
self.profileCluster.start()
jobservers=[server.strip() for server in config.get('jobqueue','hosts').split(",")]
jobports=[int(port) for port in config.get('jobqueue', 'ports').split(',')]
self.jobCluster=RedisCluster(zip(jobservers, jobports))
self.jobCluster.start()
def getSNRedis(self):
return self.snCluster
def getProfileRedis(self):
return self.profileCluster
def getJobRedis(self):
return self.jobCluster
def getGraphCache(self):
return GraphCache(self)
def getDBUserDao(self):
conn = MySQLdb.connect(host=self.config.get('mysql',"host"),
port=self.config.getint('mysql', 'port'),
user=self.config.get('mysql',"user"),
passwd=self.config.get('mysql',"passwd"),
db=self.config.get('mysql','db'),
charset="utf8")
dao = DBUserDao(conn)
return dao
def getCachedTweetDao(self):
pass
def getDBTweetDao(self):
conn = MySQLdb.connect(host=self.config.get('mysql',"host"),
port=self.config.getint('mysql', 'port'),
user=self.config.get('mysql',"user"),
passwd=self.config.get('mysql',"passwd"),
db=self.config.get('mysql','db'),
charset="utf8")
dao = DBTweetDao(conn)
return dao
def getDBCommInfoDao(self):
conn = MySQLdb.connect(host=self.config.get('mysql',"host"),
port=self.config.getint('mysql', 'port'),
user=self.config.get('mysql',"user"),
passwd=self.config.get('mysql',"passwd"),
db=self.config.get('mysql','db'),
charset="utf8")
dao = DBCommInfoDao(conn)
return dao
def getTagStatsDao(self):
conn = MySQLdb.connect(host=self.config.get('mysql',"host"),
port=self.config.getint('mysql', 'port'),
user=self.config.get('mysql',"user"),
passwd=self.config.get('mysql',"passwd"),
db=self.config.get('mysql','db'),
charset="utf8")
ret= ChainedDao([RedisTagDao(self.profileCluster), DBTagDao(conn)])#
return ret
def getCachedCrawlUserDao(self):
ret= ChainedDao([RedisUserDao(self.profileCluster), self.getDBUserDao(),
UserDataCrawlerDao(WeiboCrawler(TokenManager(self.config)), self.getTagStatsDao())])#
return ret
def getCachedCrawlSNDao(self):
ret= ChainedDao([RedisSocialDao(self.snCluster),
UserDataCrawlerDao(WeiboCrawler(TokenManager(self.config)), self.getTagStatsDao())])#
return ret
def getCachedCrawlTweetDao(self):
ret= ChainedDao([TweetCrawlerDao(WeiboCrawler(TokenManager(self.config)))])
return ret
def getClusterStateDao(self):
conn = MySQLdb.connect(host=self.config.get('mysql',"host"),
port=self.config.getint('mysql', 'port'),
user=self.config.get('mysql',"user"),
passwd=self.config.get('mysql',"passwd"),
db=self.config.get('mysql','db'),
charset="utf8")
dao = ClusterStateDao(conn)
return dao
def getUserNeighComms(self, uid):
redis = RedisCluster.getRedis(uid, NEIGH_COMMS_DB)
return redis.hgetall(uid)
def getUserNeighComms(self, uid):
redis = RedisCluster.getRedis(uid, NEIGH_COMMS_DB)
return redis.hgetall(uid)