def calc_count():
WordCount.logger.info("start to count words")
ip = Config.getProperty('mongo', 'addr')
port = int(Config.getProperty('mongo', 'port'))
client = pymongo.MongoClient(ip, port)
db = client.spiderDB
collection = db.spider
collection.map_reduce(WordCount.mapper, WordCount.reducer, out="result", full_response=True)
class MemcacheUtil:
url = Config.getProperty("memcached", "addr")
@staticmethod
def get(key):
mc = memcache.Client([MemcacheUtil.url])
return mc.get(key)
@staticmethod
def add(key, value):
mc = memcache.Client([MemcacheUtil.url])
return mc.add(key, value)
@staticmethod
def set(key, value):
mc = memcache.Client([MemcacheUtil.url])
return mc.set(key, value)
@staticmethod
def delete(key):
try:
mc = memcache.Client([MemcacheUtil.url])
mc.delete(key)
except Exception as e:
print e
@staticmethod
def clean():
mc = memcache.Client([MemcacheUtil.url])
return mc.flush_all()
def getlogger(name):
logger = logging.getLogger(name)
logger.setLevel(LogFactory.__get_logger_level(Config.getProperty("logger", "level")))
return logger
def getlogger(name):
logger = logging.getLogger(name)
logger.setLevel(
LogFactory.__get_logger_level(Config.getProperty(
"logger", "level")))
return logger
def __get_data_statics_collection(client):
db = client.get_database(Config.getProperty('mongo','resultdb'))
cl = db.get_collection(Config.getProperty('mongo','statisticCollection'))
return cl
def __get_client():
ip = Config.getProperty('mongo', 'addr')
port = int(Config.getProperty('mongo', 'port'))
client = pymongo.MongoClient(ip, port)
return client
def __get_data_statics_collection(client):
db = client.get_database(Config.getProperty('mongo', 'resultdb'))
cl = db.get_collection(
Config.getProperty('mongo', 'statisticCollection'))
return cl
def __get_client():
ip = Config.getProperty('mongo', 'addr')
port = int(Config.getProperty('mongo', 'port'))
client = pymongo.MongoClient(ip, port)
return client