def __init__(self):
try:
cparser = confparser.confdictory('backinfo')
except:
cparser = {"backpath": "/home/chemical/workspace/python/se/backup"}
if cparser.has_key('backpath'):
self.backpath = cparser['backpath']
else:
return
def __init__(self):
try:
cparser = confparser.confdictory('backinfo')
except:
cparser = {"backpath":"/home/chemical/workspace/python/se/backup"}
if cparser.has_key('backpath'):
self.backpath = cparser['backpath']
else:
return
def warning(loginfo={}):
try:
confdictory = confparser.confdictory('log')
except:
#默认log配置
confdictory = {'logpath':'log','logfile':'se.log','loglevel':'debug'}
logpath = ''
noticelog = ''
warninglog = ''
loginfo = loginfo
nowstr = strftime('%Y%m%d%H')
noticefile = ''
warningpath = ''
if confdictory.has_key('logpath'):
logpath = confdictory['logpath']
if confdictory.has_key('logfile'):
noticefile = confdictory['logfile']
warningpath = noticefile + '.' + str(nowstr) + '.wf'
warningfile = logpath + '/' + warningpath
try:
frame = _getframe(1)
except:
frame = _getframe()
curfile = frame.f_code.co_filename
linenum = frame.f_lineno
now = strftime('%Y-%m-%d %H:%M:%S')
logstr = '[level] warning [time] ' + str(now) + ' [fileinfo] ' + os.path.abspath(curfile) + ':' + str(linenum)
if isinstance(loginfo,str):
logstr += ' [msg] ' + loginfo
elif isinstance(loginfo,dict):
for info in loginfo:
if isinstance(loginfo[info],str):
logstr += ' [' + info + '] ' + loginfo[info]
else:
try:
logstr += ' [' + info + '] (' + str(loginfo[info]) + ')'
except:
logstr += ' [' + info + '] Error Format'
else:
logstr += ' [msg] (' + str(loginfo) + ')'
logstr += '\n'
if warningfile != None:
new_link = 0
if not os.path.exists(warningfile) or not os.path.isfile(warningfile):
new_link = 1
f_handler = open(warningfile,'a+')
f_handler.write(logstr)
f_handler.close()
warninglink = logpath + '/' + noticefile + '.wf'
if not os.path.islink(warninglink) or new_link == 1:
if os.path.islink(warninglink):
os.unlink(warninglink)
os.symlink(warningpath,warninglink)
def __init__(self):
self.fromurl = ''
self.linklist = []
self.parse_content = ''
self.content = ''
self.wordlist = []
confdictory = confparser.confdictory('database')
if len(confdictory) == 0:
logger.warning('config file load error')
return
self.dbcontrol = dbcontrol(confdictory['user'], \
confdictory['password'])
def __init__(self):
self.fromurl = ''
self.linklist = []
self.parse_content = ''
self.content = ''
self.wordlist = []
confdictory = confparser.confdictory('database')
if len(confdictory) == 0:
logger.warning('config file load error')
return
self.dbcontrol = dbcontrol(confdictory['user'], \
confdictory['password'])
def __init__(self, querywords):
if len(querywords.strip()) == 0 or not isinstance(querywords, str):
logger.warning('querywords invalid!')
return
self.querywords = querywords.strip().split(' ')
self.query = query(querywords)
self.resultlist = []
confmap = confparser.confdictory('backinfo')
buildmap = confparser.confdictory('build')
if confmap == None or buildmap == None:
log.warning('config error for backinfo section')
return None
if not confmap.has_key('backpath'):
self.backpath = 'backup'
else:
self.backpath = confmap['backpath']
if not confmap.has_key('rawtext'):
self.rawtext = '.file'
else:
self.rawtext = confmap['rawtext']
if not confmap.has_key('rawhtml'):
self.rawhtml = '.html'
else:
self.rawhtml = confmap['backtext']
if not confmap.has_key('backtext'):
self.backtext = '.file.zip'
else:
self.backtext = confmap['backtext']
if not confmap.has_key('backhtml'):
self.backhtml = '.htm.zip'
else:
self.backhtml = confmap['backhtml']
if not buildmap.has_key('block'):
self.block = 128
else:
self.block = int(buildmap['block'])
def __init__(self,querywords):
if len(querywords.strip()) == 0 or not isinstance(querywords,str):
logger.warning('querywords invalid!')
return
self.querywords = querywords.strip().split(' ')
self.query = query(querywords)
self.resultlist = []
confmap = confparser.confdictory('backinfo')
buildmap = confparser.confdictory('build')
if confmap == None or buildmap == None:
log.warning('config error for backinfo section')
return None
if not confmap.has_key('backpath'):
self.backpath = 'backup'
else:
self.backpath = confmap['backpath']
if not confmap.has_key('rawtext'):
self.rawtext = '.file'
else:
self.rawtext = confmap['rawtext']
if not confmap.has_key('rawhtml'):
self.rawhtml = '.html'
else:
self.rawhtml = confmap['backtext']
if not confmap.has_key('backtext'):
self.backtext = '.file.zip'
else:
self.backtext = confmap['backtext']
if not confmap.has_key('backhtml'):
self.backhtml = '.htm.zip'
else:
self.backhtml = confmap['backhtml']
if not buildmap.has_key('block'):
self.block = 128
else:
self.block = int(buildmap['block'])
def debug(loginfo={}):
try:
confdictory = confparser.confdictory('log')
except:
#默认log配置
confdictory = {
'logpath': 'log/',
'logfile': 'se.log',
'loglevel': 'debug'
}
logpath = ''
debuglog = ''
loginfo = loginfo
nowstr = strftime('%Y%m%d%H')
if confdictory.has_key('logpath'):
logpath = confdictory['logpath']
if confdictory.has_key('logfile'):
confpath = confdictory['logfile']
debugpath = confpath + '.' + str(nowstr) + '.dbg'
debugfile = logpath + '/' + debugpath
try:
frame = _getframe(1)
except:
frame = _getframe()
curfile = frame.f_code.co_filename
linenum = frame.f_lineno
now = strftime('%Y-%m-%d %H:%M:%S')
logstr = '[level] debug [time] ' + str(
now) + ' [fileinfo] ' + os.path.abspath(curfile) + ':' + str(
linenum)
if isinstance(loginfo, str):
logstr += ' [msg] ' + loginfo
elif isinstance(loginfo, dict):
for info in loginfo:
if isinstance(loginfo[info], str):
logstr += ' [' + info + '] ' + loginfo[info]
else:
try:
logstr += ' [' + info + '] (' + str(
loginfo[info]) + ')'
except:
logstr += ' [' + info + '] Error Format'
else:
logstr += ' [msg] (' + str(loginfo) + ')'
logstr += '\n'
if debugfile != None:
f_handler = open(debugfile, 'a+')
f_handler.write(logstr)
f_handler.close()
def __init__(self,query_words):
if isinstance(query_words,str):
self.wordlist = query_words.split(' ')
#reslist 存放wordlist的对应结果
self.reslist = []
confdictory = confparser.confdictory('database')
if not confdictory.has_key('user'):# or !confdictory.has_key['dbpass']:
logger.warning('config parser can not be loaded')
return None
self.engine = dbcontrol(confdictory['user'],confdictory['password'])
if self.engine == None:
logger.warning({'msg':'database connect error','dbname':confdictory['dbuser'],'dbpass':confdictory['dbpass']})
return None
else:
logger.warning('query words format error')
def __init__(self):
#加载服务器信息
os.chdir(os.getcwd())
servermap = confparser.confdictory('server')
if servermap == None or len(servermap) == 0:
logger.warning("config file load error")
return None
if servermap.has_key('address'):
self.addr = servermap['address']
else:
self.addr = 'localhost'
if servermap.has_key('port'):
self.port = int(servermap['port'])
else:
self.port = 9898
if servermap.has_key('listennum'):
self.listennum = int(servermap['listennum'])
else:
self.listennum = 10
if servermap.has_key('timeout'):
self.timeout = int(servermap['timeout'])
else:
self.timeout = 2
if servermap.has_key('pidpath'):
self.pidpath = servermap['pidpath']
else:
self.pidpath = "tmp"
if servermap.has_key('pidfile'):
self.pidfile = servermap['pidfile']
else:
self.pidfile = "server.pid"
if servermap.has_key('recvlen'):
self.recvlen = int(servermap['recvlen'])
else:
self.recvlen = 1024
if servermap.has_key('recvbuf'):
self.recvbuf = int(servermap['recvbuf'])
else:
self.recvbuf = 4096
if servermap.has_key('sendbuf'):
self.sendbuf = int(servermap['sendbuf'])
else:
self.sendbuf = 8192
self.__setpid()
def __init__(self):
#加载服务器信息
os.chdir(os.getcwd())
servermap = confparser.confdictory('server')
if servermap == None or len(servermap) == 0:
logger.warning("config file load error")
return None
if servermap.has_key('address'):
self.addr = servermap['address']
else:
self.addr = 'localhost'
if servermap.has_key('port'):
self.port = int(servermap['port'])
else:
self.port = 9898
if servermap.has_key('listennum'):
self.listennum = int(servermap['listennum'])
else:
self.listennum = 10
if servermap.has_key('timeout'):
self.timeout = int(servermap['timeout'])
else:
self.timeout = 2
if servermap.has_key('pidpath'):
self.pidpath = servermap['pidpath']
else:
self.pidpath = "tmp"
if servermap.has_key('pidfile'):
self.pidfile = servermap['pidfile']
else:
self.pidfile = "server.pid"
if servermap.has_key('recvlen'):
self.recvlen = int(servermap['recvlen'])
else:
self.recvlen = 1024
if servermap.has_key('recvbuf'):
self.recvbuf = int(servermap['recvbuf'])
else:
self.recvbuf = 4096
if servermap.has_key('sendbuf'):
self.sendbuf = int(servermap['sendbuf'])
else:
self.sendbuf = 8192
self.__setpid()
def debug(loginfo={}):
try:
confdictory = confparser.confdictory('log')
except:
#默认log配置
confdictory = {'logpath':'log/','logfile':'se.log','loglevel':'debug'}
logpath = ''
debuglog = ''
loginfo = loginfo
nowstr = strftime('%Y%m%d%H')
if confdictory.has_key('logpath'):
logpath = confdictory['logpath']
if confdictory.has_key('logfile'):
confpath = confdictory['logfile']
debugpath = confpath + '.' + str(nowstr) + '.dbg'
debugfile = logpath + '/' + debugpath
try:
frame = _getframe(1)
except:
frame = _getframe()
curfile = frame.f_code.co_filename
linenum = frame.f_lineno
now = strftime('%Y-%m-%d %H:%M:%S')
logstr = '[level] debug [time] ' + str(now) + ' [fileinfo] ' + os.path.abspath(curfile) + ':' + str(linenum)
if isinstance(loginfo,str):
logstr += ' [msg] ' + loginfo
elif isinstance(loginfo,dict):
for info in loginfo:
if isinstance(loginfo[info],str):
logstr += ' [' + info + '] ' + loginfo[info]
else:
try:
logstr += ' [' + info + '] (' + str(loginfo[info]) + ')'
except:
logstr += ' [' + info + '] Error Format'
else:
logstr += ' [msg] (' + str(loginfo) + ')'
logstr += '\n'
if debugfile != None:
f_handler = open(debugfile,'a+')
f_handler.write(logstr)
f_handler.close()
def __init__(self, query_words):
if isinstance(query_words, str):
self.wordlist = query_words.split(' ')
#reslist 存放wordlist的对应结果
self.reslist = []
confdictory = confparser.confdictory('database')
if not confdictory.has_key(
'user'): # or !confdictory.has_key['dbpass']:
logger.warning('config parser can not be loaded')
return None
self.engine = dbcontrol(confdictory['user'],
confdictory['password'])
if self.engine == None:
logger.warning({
'msg': 'database connect error',
'dbname': confdictory['dbuser'],
'dbpass': confdictory['dbpass']
})
return None
else:
logger.warning('query words format error')
def __init__(self):
confdict = confparser.confdictory('rank')
if len(confdict) == 0:
logger.warning('config file load error')
self.weightdict = {}
#根据文件配置生成排名函数
if confdict.has_key('frequence'):
self.weightdict['frequence'] = {}
self.weightdict['frequence']['caller'] = self.frequence
self.weightdict['frequence']['weight'] = int(confdict['frequence'])
if confdict.has_key('position'):
self.weightdict['position'] = {}
self.weightdict['position']['caller'] = self.position
self.weightdict['position']['weight'] = int(confdict['position'])
if confdict.has_key('distance'):
self.weightdict['distance'] = {}
self.weightdict['distance']['caller'] = self.querydistance
self.weightdict['distance']['weight'] = int(confdict['distance'])
if confdict.has_key('titleweight'):
self.weightdict['titleweight'] = {}
self.weightdict['titleweight']['caller'] = self.titleweight
self.weightdict['titleweight']['weight'] = int(confdict['distance'])
def __init__(self):
confdict = confparser.confdictory('rank')
if len(confdict) == 0:
logger.warning('config file load error')
self.weightdict = {}
#根据文件配置生成排名函数
if confdict.has_key('frequence'):
self.weightdict['frequence'] = {}
self.weightdict['frequence']['caller'] = self.frequence
self.weightdict['frequence']['weight'] = int(confdict['frequence'])
if confdict.has_key('position'):
self.weightdict['position'] = {}
self.weightdict['position']['caller'] = self.position
self.weightdict['position']['weight'] = int(confdict['position'])
if confdict.has_key('distance'):
self.weightdict['distance'] = {}
self.weightdict['distance']['caller'] = self.querydistance
self.weightdict['distance']['weight'] = int(confdict['distance'])
if confdict.has_key('titleweight'):
self.weightdict['titleweight'] = {}
self.weightdict['titleweight']['caller'] = self.titleweight
self.weightdict['titleweight']['weight'] = int(
confdict['distance'])
def notice(loginfo={}):
try:
confdictory = confparser.confdictory('spider')
except:
#默认log配置
confdictory = {
'slogpath': 'log',
'slogfile': 'spider.log',
'sloglevel': 'debug'
}
logpath = ''
noticelog = ''
loginfo = loginfo
nowstr = strftime('%Y%m%d%H')
confpath = ''
noticepath = ''
if confdictory.has_key('slogpath'):
logpath = confdictory['slogpath']
if confdictory.has_key('slogfile'):
confpath = confdictory['slogfile']
noticepath = confpath + '.' + str(nowstr)
noticefile = logpath + '/' + noticepath
try:
frame = _getframe(1)
except:
frame = _getframe()
curfile = frame.f_code.co_filename
linenum = frame.f_lineno
now = strftime('%Y-%m-%d %H:%M:%S')
logstr = '[level] notice [time] ' + str(
now) + ' [fileinfo] ' + os.path.abspath(curfile) + ':' + str(
linenum)
if isinstance(loginfo, str):
logstr += ' [msg] ' + loginfo
elif isinstance(loginfo, dict):
for info in loginfo:
if isinstance(loginfo[info], str):
logstr += ' [' + info + '] ' + loginfo[info]
else:
try:
logstr += ' [' + info + '] (' + str(
loginfo[info]) + ')'
except:
logstr += ' [' + info + '] Error Format'
else:
logstr += ' [msg] (' + str(loginfo) + ')'
logstr += '\n'
if noticefile != None:
new_link = 0
#创建软链接
if not os.path.exists(noticefile) or not os.path.isfile(
noticefile):
new_link = 1
f_handler = open(noticefile, 'a+')
f_handler.write(logstr)
f_handler.close()
noticelink = logpath + '/' + confpath
if not os.path.islink(noticelink) or new_link == 1:
if os.path.islink(noticelink):
os.unlink(noticelink)
os.symlink(noticepath, noticelink)
