def addRecord(self, ip):
'''add a new connection to redis'''
# set searchkey
globalLog.getError().log(globalLog.DEBUG, \
"searchkey:%s"%(ip))
# get info from redis and store info into redis
return True if self._addRecord(ip) >= self._maxnum else False
def _writeReject(self, data):
'''write attack information into file'''
# create attackfile
try:
if os.path.exists(BLACK_FILE) is False:
open(BLACK_FILE, 'w').close()
if os.path.exists(BLACK_PIPE) is False:
os.mkfifo(BLACK_PIPE)
except Exception as msg:
globalLog.getError().log(
globalLog.WARNING,
'%s:Handle attack file %s.' % (msg, BLACK_FILE))
return
# write info into attack file(pipe and normal file)
try:
# Whether existed blacklist ip
with open(BLACK_FILE, 'r') as reFd:
strbuf = reFd.read()
if data in strbuf:
return
# write black into file
with open(BLACK_FILE, 'a') as inFd:
inFd.write(data + '\n')
# write into pipe
self._attackPip.write(data)
self._attackPip.flush()
except Exception as msg:
globalLog.getError().log(globalLog.WARNING,
'Warning:replace file failed(%s)' % (msg))
def stop(self):
"""
Stop the daemon
"""
# Get the pid from the pidfile
try:
pf = file(self.pidfile, 'r')
pid = int(pf.read().strip())
pf.close()
except IOError:
pid = None
if not pid:
message = "pidfile %s does not exist. Daemon not running?\n"
globalLog.getError().log(globalLog.ERROR, message % self.pidfile)
return # not an error in a restart
# Try killing the daemon process
try:
while True:
os.kill(pid, SIGTERM)
time.sleep(0.1)
except OSError as err:
err = str(err)
if err.find("No such process") > 0:
if os.path.exists(self.pidfile):
os.remove(self.pidfile)
else:
globalLog.getError().log(globalLog.ERROR,
'Daemon stop:%s' % err)
sys.exit(1)
def stop(self):
"""
Stop the daemon
"""
# Get the pid from the pidfile
try:
pf = file(self.pidfile,'r')
pid = int(pf.read().strip())
pf.close()
except IOError:
pid = None
if not pid:
message = "pidfile %s does not exist. Daemon not running?\n"
globalLog.getError().log(globalLog.ERROR, message % self.pidfile)
return # not an error in a restart
# Try killing the daemon process
try:
while 1:
os.kill(pid, SIGTERM)
time.sleep(0.1)
except OSError, err:
err = str(err)
if err.find("No such process") > 0:
if os.path.exists(self.pidfile):
os.remove(self.pidfile)
else:
globalLog.getError().log(globalLog.ERROR, 'Daemon stop:%s'%err)
sys.exit(1)
def connectDb(self):
'''connect DB'''
try:
self._db = MySQLdb.connect(self._address, self._user, self._passwd, self._db)
self._cursor = self._db.cursor()
except Exception,msg:
globalLog.getError().log(globalLog.ERROR, \
'Connect SqlDB failed, msg:%s'%(msg))
return
def _handleModuleRedis(self, len, data):
'''handle redis package'''
globalLog.getError().log(globalLog.DEBUG,
"HandleRedis:blackip:%s" % data)
#self._redisConn.addRecord(info[0], info[1])
if self._redisConn.addRecord(data):
globalLog.getError().log(globalLog.INFO,
"Write blacklist:%s" % data)
self._writeReject(data)
def _handleModuleRedis(self, len, data):
'''handle redis package'''
globalLog.getError().log(globalLog.DEBUG, \
"HandleRedis:blackip:%s"%data)
#self._redisConn.addRecord(info[0], info[1])
if self._redisConn.addRecord(data):
globalLog.getError().log(globalLog.INFO, \
"Write blacklist:%s"%data)
self._writeReject(data)
def innerFunc(*args, **kw):
try:
return func(*args, **kw)
except Exception,msg:
traceList = traceback.extract_tb(sys.exc_traceback)
for (file,lineno, funcname,text) in traceList:
globalLog.getError().log(globalLog.ERROR, \
"Occur error, func:%s,lineno:%s, msg:%s"%(funcname, lineno, msg))
return None
def innerFunc(*args, **kw):
try:
return func(*args, **kw)
except Exception as msg:
traceList = traceback.extract_tb(sys.exc_info()[2])
for (file, lineno, funcname, text) in traceList:
globalLog.getError().log(
globalLog.ERROR, "Occur error, func:%s,lineno:%s, msg:%s" %
(funcname, lineno, msg))
sys.exit(-1)
def __init__(self, host='127.0.0.1', port=6379, maxnum = 2000000):
'''init'''
try:
self._client = redis.Redis(host, port, db=0)
self._pipeline = self._client.pipeline() # transation
except Exception:
globalLog.getError().log(globalLog.ERROR, 'connect redis server failed!')
sys.exit(-1)
self._maxnum = maxnum
# redis key which store black list
self._attackKey = 'nginx.blacklist'
def __init__(self, host='127.0.0.1', port=6379, maxnum=2000000):
'''init'''
try:
self._client = redis.Redis(host, port, db=0)
self._pipeline = self._client.pipeline() # transation
except Exception:
globalLog.getError().log(globalLog.ERROR,
'connect redis server failed!')
sys.exit(-1)
self._maxnum = maxnum
# redis key which store black list
self._attackKey = 'nginx.blacklist'
def _createPipe(self, filename):
'''Create a pipe'''
try:
if os.path.exists(filename) is False:
os.mkfifo(filename)
self._attackPip = open(filename, 'w')
globalLog.getError().log(globalLog.DEBUG, \
'Create pipe file successful')
except (OSError, Exception), msg:
globalLog.getError().log(globalLog.ERROR, \
'Create and Open FIFO failed, %s'%msg)
sys.exit(-1)
def _createPipe(self, filename):
'''Create a pipe'''
try:
if os.path.exists(filename) is False:
os.mkfifo(filename)
self._attackPip = open(filename, 'w')
globalLog.getError().log(globalLog.DEBUG,
'Create pipe file successful')
except (OSError, Exception) as msg:
globalLog.getError().log(globalLog.ERROR,
'Create and Open FIFO failed, %s' % msg)
sys.exit(-1)
def _writeReject(self, data):
'''write attack information into file'''
# create attackfile
try:
if os.path.exists(BLACK_FILE) is False:
open(BLACK_FILE, 'w').close()
if os.path.exists(BLACK_PIPE) is False:
os.mkfifo(BLACK_PIPE)
except Exception,msg:
globalLog.getError().log(globalLog.WARNING, \
'%s:Handle attack file %s.'%(msg, BLACK_FILE))
return
def _bindSocket(self, ip, port):
'''bind socket'''
address = (ip, port)
try:
self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self._sock.bind(address)
self._sock.setblocking(0)
self._sock.listen(10)
except Exception, msg:
globalLog.getError().log(globalLog.ERROR, \
'AgentServer:bind socket %s failed, %s'%(ip,msg))
sys.exit(-1)
def daemonize(self):
"""
do the UNIX double-fork magic, see Stevens' "Advanced
Programming in the UNIX Environment" for details (ISBN 0201563177)
"""
# do first fork
try:
pid = os.fork()
if pid > 0:
sys.exit(0)
except OSError, e:
globalLog.getError().log(globalLog.ERROR,
"fork #1 failed: %d (%s)\n" % (e.errno, e.strerror))
sys.exit(1)
def _addRecord(self, searchkey):
'''add a new connection to redis'''
# add command to pipe
if not self._client.get(searchkey):
self._pipeline.setex(searchkey, 0, 60)
else:
self._pipeline.incr(searchkey)
self._pipeline.get(searchkey)
# execute command
try:
return int(self._pipeline.execute()[-1])
except Exception:
globalLog.getError().log(
globalLog.ERROR, 'AgentServer:execute redis command failed.')
return self._maxnum
def _addRecord(self, searchkey):
'''add a new connection to redis'''
# add command to pipe
if not self._client.get(searchkey):
self._pipeline.setex(searchkey, 0, 60)
else:
self._pipeline.incr(searchkey)
self._pipeline.get(searchkey)
# execute command
try:
return int(self._pipeline.execute()[-1])
except Exception:
globalLog.getError().log(globalLog.ERROR, \
'AgentServer:execute redis command failed.')
return self._maxnum
def _bindSocket(self, ip, port):
'''bind socket'''
address = (ip, port)
try:
self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self._sock.bind(address)
self._sock.setblocking(0)
self._sock.listen(10)
except Exception as msg:
globalLog.getError().log(
globalLog.ERROR,
'AgentServer:bind socket %s failed, %s' % (ip, msg))
sys.exit(-1)
globalLog.getError().log(globalLog.DEBUG, 'Bind %s successful.' % ip)
def executeSql(self, sql, isReturn=False):
'''execute sql command'''
try:
self.execute(sql)
self._db.commit()
if not isReturn:
return
results = self._cursor.fetchall()
for row in results:
globalLog.getError.log(globalLog.ERROR, \
'row0=%s'%(row[0]))
except Exception,msg:
self._db.rollbak()
globalLog.getError().log(globalLog.ERROR, \
'Execute sql failed, sql=%s, msg:%s'%(sql, msg))
def waitConnection(self, poller, readOnly, fdToSocket, timeout):
''''''
# poll timeout
events = poller.poll(timeout)
for fd, flag in events:
# retrieve the actual socket from its file descriptor
s = fdToSocket[fd]
# handle inputs
if (flag & (select.POLLIN | select.POLLPRI)) and \
not self._receivePackage(poller, s, fdToSocket, readOnly):
poller.unregister(s)
s.close()
del fdToSocket[fd]
elif flag & (select.POLLHUP | select.POLLERR):
globalLog.getError().log(globalLog.ERROR, 'Client close connection')
poller.unregister(s)
s.close()
def _receivePackage(self, poller, conn, fdToSocket, readOnly):
'''receive package'''
try:
if conn is self._sock:
# add clifd to events
globalLog.getError().log(globalLog.DEBUG, 'A new connection come on.')
conn, cliAddr = conn.accept()
conn.setblocking(0)
fdToSocket[conn.fileno()] = conn
poller.register(conn, readOnly)
return True
else:
# receive data
return False if not self._recvPackageFromClient(conn) else True
except Exception, msg:
globalLog.getError().log(globalLog.ERROR, msg)
return False
def _receivePackage(self, poller, conn, fdToSocket, readOnly):
'''receive package'''
try:
if conn is self._sock:
# add clifd to events
globalLog.getError().log(globalLog.DEBUG,
'A new connection come on.')
conn, cliAddr = conn.accept()
conn.setblocking(0)
fdToSocket[conn.fileno()] = conn
poller.register(conn, readOnly)
return True
else:
# receive data
return False if not self._recvPackageFromClient(conn) else True
except Exception as msg:
globalLog.getError().log(globalLog.ERROR, msg)
return False
def waitConnection(self, poller, readOnly, fdToSocket, timeout):
''''''
# poll timeout
events = poller.poll(timeout)
for fd, flag in events:
# retrieve the actual socket from its file descriptor
s = fdToSocket[fd]
# handle inputs
if (flag & (select.POLLIN | select.POLLPRI)) and \
not self._receivePackage(poller, s, fdToSocket, readOnly):
poller.unregister(s)
s.close()
del fdToSocket[fd]
elif flag & (select.POLLHUP | select.POLLERR):
globalLog.getError().log(globalLog.ERROR,
'Client close connection')
poller.unregister(s)
s.close()
def handle(self):
'''waitting client connection'''
# commonly used flag sets
timeout = 1000
readOnly = (select.POLLIN | select.POLLPRI|
select.POLLHUP | select.POLLERR)
# set up the poller
self.poller = select.poll()
self.poller.register(self._sock, readOnly)
# map file descirptors to socket objects
self.fdToSocket = {
self._sock.fileno():self._sock,
}
# communicate
while True:
try:
self.waitConnection(self.poller, readOnly, self.fdToSocket, timeout)
except Exception,msg:
globalLog.getError().log(globalLog.ERROR, \
'Exception in server handle:%s'%msg);
def handle(self):
'''waitting client connection'''
# commonly used flag sets
timeout = 1000
readOnly = (select.POLLIN | select.POLLPRI | select.POLLHUP
| select.POLLERR)
# set up the poller
self.poller = select.poll()
self.poller.register(self._sock, readOnly)
# map file descirptors to socket objects
self.fdToSocket = {
self._sock.fileno(): self._sock,
}
# communicate
while True:
try:
self.waitConnection(self.poller, readOnly, self.fdToSocket,
timeout)
except Exception as msg:
globalLog.getError().log(globalLog.ERROR,
'Exception in server handle:%s' % msg)
def start(self):
"""
Start the daemon
"""
# Check for a pidfile to see if the daemon already runs
try:
pf = file(self.pidfile, 'r')
pid = int(pf.read().strip())
pf.close()
except IOError:
pid = None
# existed daemon process, quit
if pid:
message = "pidfile %s already exist. Daemon already running?\n"
globalLog.getError().log(globalLog.ERROR, message % self.pidfile)
sys.exit(1)
# Start the daemon
self.daemonize()
# run
self.run()
def start(self):
"""
Start the daemon
"""
# Check for a pidfile to see if the daemon already runs
try:
pf = file(self.pidfile,'r')
pid = int(pf.read().strip())
pf.close()
except IOError:
pid = None
# existed daemon process, quit
if pid:
message = "pidfile %s already exist. Daemon already running?\n"
globalLog.getError().log(globalLog.ERROR, message % self.pidfile)
sys.exit(1)
# Start the daemon
self.daemonize()
# run
self.run()
def daemonize(self):
"""
do the UNIX double-fork magic, see Stevens' "Advanced
Programming in the UNIX Environment" for details (ISBN 0201563177)
"""
# do first fork
try:
pid = os.fork()
if pid > 0:
sys.exit(0)
except OSError as e:
globalLog.getError().log(
globalLog.ERROR,
"fork #1 failed: %d (%s)\n" % (e.errno, e.strerror))
sys.exit(1)
# decouple from parent environment
os.chdir("/")
os.setsid()
os.umask(0)
# do second fork
try:
pid = os.fork()
if pid > 0:
sys.exit(0)
except OSError as e:
globalLog.getError().log(
globalLog.ERROR,
"fork #2 failed: %d (%s)\n" % (e.errno, e.strerror))
sys.exit(1)
# redirect standard file descriptors
sys.stdout.flush()
sys.stderr.flush()
si = file(self.stdin, 'r')
so = file(self.stdout, 'a+')
se = file(self.stderr, 'a+', 0)
os.dup2(si.fileno(), sys.stdin.fileno())
os.dup2(so.fileno(), sys.stdout.fileno())
os.dup2(se.fileno(), sys.stderr.fileno())
# registrem destructor function
atexit.register(self.delpid)
# write pidfile
try:
fd = open(self.pidfile, 'w+')
fd.write('%s\n' % (str(os.getpid())))
fd.close()
except IOError as msg:
globalLog.getError().log(globalLog.ERROR,
"%s:open pidfile failed." % msg)
sys.exit(1)
def addRecord(self, ip):
'''add a new connection to redis'''
# set searchkey
globalLog.getError().log(globalLog.DEBUG, "searchkey:%s" % (ip))
# get info from redis and store info into redis
return True if self._addRecord(ip) >= self._maxnum else False
def sigInt(self, signum, frame):
'''SIGINT handle'''
globalLog.getError().log(globalLog.ERROR, "Receive SIGINT, eixt process")
from basepackage.baselog import globalLog
if __name__ == '__main__':
'''test'''
'''测试AgentServer通信机制'''
server_address = '127.0.0.1'
port = 8008
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# 连接服务器
try:
sock.connect((server_address, port))
except socket.error as msg:
sock.close()
print(msg)
globalLog.getError().log(globalLog.ERROR, '%s' % msg)
# 发送报文信息
for i in range(1, 10000):
client_ip = '192.168.168.168'
token = '0' if i % 2010 else '20150724205220-gelgessssssjsjsjsjsjsjsjsjs%d' % (
i)
data = ' '.join([client_ip, token])
ptype = '00'
length = '%04d' % len(data)
package = ''.join([ptype, length, data])
print('%s ----- %d' % (package, i))
try:
sock.send(package)
except socket.error as msg:
print(msg)
def _recvPackageFromClient(self, conn):
'''conmmnicate between server and client'''
preLength = 8 # len(type + data_len)，结构体本身的长度
#import pdb
# pdb.set_trace()
# receive package (hexadecimal number)
data = conn.recv(preLength)
if not data:
globalLog.getError().log(globalLog.DEBUG,
'Received Null string, client close.')
return False
elif len(data) != preLength:
globalLog.getError().log(globalLog.DEBUG,
'Package too small, drop package.')
return True
(type, totalLen) = struct.unpack('<ii', data)
globalLog.getError().log(
globalLog.DEBUG,
"Package Type:%d, PackageLeng=%d" % (type, totalLen))
# receive package(content)
data = conn.recv(totalLen)
if not data:
globalLog.getError().log(
globalLog.ERROR, 'Received Null string, perhaps client close.')
return False
elif len(data) != totalLen:
globalLog.getError().log(globalLog.DEBUG,
'Package too small, drop package.')
return True
# handle
globalLog.getError().log(globalLog.DEBUG, "Pacakge Content=%s" % data)
self._handleModule(type, totalLen, data)
return True
si = file(self.stdin, 'r')
so = file(self.stdout, 'a+')
se = file(self.stderr, 'a+', 0)
os.dup2(si.fileno(), sys.stdin.fileno())
os.dup2(so.fileno(), sys.stdout.fileno())
os.dup2(se.fileno(), sys.stderr.fileno())
# registrem destructor function
atexit.register(self.delpid)
# write pidfile
try:
fd = open(self.pidfile, 'w+')
fd.write('%s\n'%(str(os.getpid())))
fd.close()
except IOError,msg:
globalLog.getError().log(globalLog.ERROR, "%s:open pidfile failed."%msg)
sys.exit(1)
def __del__(self):
'''some handle after SIGTERM'''
pass
def sigInt(self, signum, frame):
'''SIGINT handle'''
globalLog.getError().log(globalLog.ERROR, "Receive SIGINT, eixt process")
#os.kill(os.getpid(), signal.SIGTERM)
def delpid(self):
'''destructor function'''
os.remove(self.pidfile)
from basepackage.baselog import globalLog
if __name__ == '__main__':
'''test'''
'''测试AgentServer通信机制'''
server_address = '127.0.0.1'
port = 8008
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# 连接服务器
try:
sock.connect((server_address, port))
except socket.error,msg:
sock.close()
print msg
globalLog.getError().log(globalLog.ERROR, '%s'%msg)
# 发送报文信息
for i in range(1,10000):
client_ip = '192.168.168.168'
token = '0' if i%2010 else '20150724205220-gelgessssssjsjsjsjsjsjsjsjs%d'%(i)
data = ' '.join([client_ip, token])
ptype = '00'
length = '%04d'%len(data)
package = ''.join([ptype, length, data])
print '%s ----- %d'%(package, i)
try:
sock.send(package)
except socket.error, msg:
print msg
globalLog.getError().log(globalLog.ERROR, '%s'%msg)
def _recvPackageFromClient(self, conn):
'''conmmnicate between server and client'''
preLength = 8# len(type + data_len)，结构体本身的长度
#import pdb
#pdb.set_trace()
# receive package (hexadecimal number)
data = conn.recv(preLength)
if not data:
globalLog.getError().log(globalLog.DEBUG, \
'Received Null string, client close.')
return False
elif len(data) != preLength:
globalLog.getError().log(globalLog.DEBUG, \
'Package too small, drop package.')
return True
(type, totalLen) = struct.unpack('<ii', data)
globalLog.getError().log(globalLog.DEBUG, \
"Package Type:%d, PackageLeng=%d"%(type, totalLen))
# receive package(content)
data = conn.recv(totalLen)
if not data:
globalLog.getError().log(globalLog.ERROR, \
'Received Null string, perhaps client close.')
return False
elif len(data) != totalLen:
globalLog.getError().log(globalLog.DEBUG, \
'Package too small, drop package.')
return True
# handle
globalLog.getError().log(globalLog.DEBUG, "Pacakge Content=%s"%data)
self._handleModule(type, totalLen, data)
return True
sys.exit(-1)
def _bindSocket(self, ip, port):
'''bind socket'''
address = (ip, port)
try:
self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self._sock.bind(address)
self._sock.setblocking(0)
self._sock.listen(10)
except Exception, msg:
globalLog.getError().log(globalLog.ERROR, \
'AgentServer:bind socket %s failed, %s'%(ip,msg))
sys.exit(-1)
globalLog.getError().log(globalLog.DEBUG, 'Bind %s successful.'%ip)
def _writeReject(self, data):
'''write attack information into file'''
# create attackfile
try:
if os.path.exists(BLACK_FILE) is False:
open(BLACK_FILE, 'w').close()
if os.path.exists(BLACK_PIPE) is False:
os.mkfifo(BLACK_PIPE)
except Exception,msg:
globalLog.getError().log(globalLog.WARNING, \
'%s:Handle attack file %s.'%(msg, BLACK_FILE))
return
# write info into attack file(pipe and normal file)
