def __init__(self,server=None,token=None,target=None):
super(Scanner, self).__init__()
self.server = server
self.token = token
self.target = target
# web接口
self.web_interface = None
if server and token:
self.web_interface = WebInterface(server,token)
# 任务
self.services = []
# 扫描结果
self.result = {}
# pluginLoaders
self.pls = []
def __init__(self, url=None, server=None, session=None):
super(Scanner, self).__init__()
#url
if url[-1] != '/':
url += '/'
self.url = url
# web server class
self.web_interface = None
if server and session:
self.web_interface = WebInterface(server, session)
m = re.match('(http[s]?)://([^:^/]+):?([^/]*)/', url)
if m:
self.http_type = m.group(1)
self.host = m.group(2)
self.ports = m.group(3)
self.ip = socket.gethostbyname(self.host)
self.domain = GetFirstLevelDomain(self.host)
else:
print 'not a valid url', url
sys.exit(0)
commonports = '21,22,23,25,110,53,67,80,443,1521,1526,3306,3389,8080,8580'
if self.ports != '':
self.ports = commonports + ',' + ports
else:
self.ports = commonports
# every plugin's input argument services
self.services = {}
self.services['url'] = self.url
self.services['host'] = self.host
self.services['ports'] = [self.ports]
self.services['http'] = []
# scan result
self.result = {}
# thread arguments
self.lock = threading.Lock()
# urls
self.urls = {}
# pluginloaders
self.pls = []
def __init__(self,conffile):
super(Scanner, self).__init__()
config = json.load(open(conffile,'r'))
# 1. init globalVar.config first
globalVar.config = config
pprint(globalVar.config['global'])
# 2.
self.server = config['global']['server']
self.token = config['global']['token']
# 注意targetname直接在config的key,而不是config['global']的key
self.targetname = config['targetname']
self.target = config['global']['target']
self.threads = int(config['global']['threads']) if config['global']['threads']!= '' else multiprocessing.cpu_count()
print 'self.threads=',self.threads,type(self.threads)
# print "config['global']['gatherdepth']=",config['global']['gatherdepth']
self.gatherdepth = int(config['global']['gatherdepth']) if config['global']['gatherdepth']!= '' else 0
# print 'self.gatherdepth=',self.gatherdepth
self.loglevel = config['global']['loglevel'] if config['global']['threads'] == '' else 'INFO'
self.args = {'loglevel':self.loglevel,'threads':self.threads,'gatherdepth':self.gatherdepth}
self.pluginargs = config['plugins']
# web接口
self.web_interface = None
if self.server and self.token:
self.web_interface = WebInterface(self.server,self.token)
# 任务
self.services = []
# 扫描结果
self.result = {}
# pluginLoaders
self.pls = []
# 3. init logging
self.loghandler = []
# log 模块,确保赋值一次
if globalVar.mainlogger is None:
globalVar.mainlogger = logging.getLogger('main')
if self.loglevel == 'DEBUG':
globalVar.mainlogger.setLevel(logging.DEBUG)
else:
globalVar.mainlogger.setLevel(logging.INFO)
# logging handler
formatter = logging.Formatter('[%(process)d] - [%(levelname)s] - %(message)s')
# 创建一个handler，用于写入日志文件
filepath = BASEDIR+'/output/log/' + genFilename(self.targetname) + '.log'
if os.path.isfile(filepath):
os.remove(filepath)
fh = logging.FileHandler(filepath,'a')
# 再创建一个handler，用于输出到控制台
ch = logging.StreamHandler()
fi = logging.Filter('main')
fh.addFilter(fi)
ch.addFilter(fi)
fh.setFormatter(formatter)
ch.setFormatter(formatter)
self.loghandler.append(ch)
self.loghandler.append(fh)
self._initLogging()
globalVar.mainlogger.info('[*] Start a new scan')
globalVar.mainlogger.info('\tserver\t=%s' % self.server)
globalVar.mainlogger.info('\ttoken\t=%s' % self.token)
globalVar.mainlogger.info('\ttarget\t=%s' % self.target)
globalVar.mainlogger.info('\tthreads\t=%d' % self.threads)
# 注意：不能通过以下的方式进行清空
# globalVar.undone_targets = []
tmpundone = copy.deepcopy(globalVar.undone_targets)
for each_target in tmpundone:
globalVar.undone_targets.remove(each_target)
class Scanner(object):
"""docstring for Scanner"""
def __init__(self,conffile):
super(Scanner, self).__init__()
config = json.load(open(conffile,'r'))
# 1. init globalVar.config first
globalVar.config = config
pprint(globalVar.config['global'])
# 2.
self.server = config['global']['server']
self.token = config['global']['token']
# 注意targetname直接在config的key,而不是config['global']的key
self.targetname = config['targetname']
self.target = config['global']['target']
self.threads = int(config['global']['threads']) if config['global']['threads']!= '' else multiprocessing.cpu_count()
print 'self.threads=',self.threads,type(self.threads)
# print "config['global']['gatherdepth']=",config['global']['gatherdepth']
self.gatherdepth = int(config['global']['gatherdepth']) if config['global']['gatherdepth']!= '' else 0
# print 'self.gatherdepth=',self.gatherdepth
self.loglevel = config['global']['loglevel'] if config['global']['threads'] == '' else 'INFO'
self.args = {'loglevel':self.loglevel,'threads':self.threads,'gatherdepth':self.gatherdepth}
self.pluginargs = config['plugins']
# web接口
self.web_interface = None
if self.server and self.token:
self.web_interface = WebInterface(self.server,self.token)
# 任务
self.services = []
# 扫描结果
self.result = {}
# pluginLoaders
self.pls = []
# 3. init logging
self.loghandler = []
# log 模块,确保赋值一次
if globalVar.mainlogger is None:
globalVar.mainlogger = logging.getLogger('main')
if self.loglevel == 'DEBUG':
globalVar.mainlogger.setLevel(logging.DEBUG)
else:
globalVar.mainlogger.setLevel(logging.INFO)
# logging handler
formatter = logging.Formatter('[%(process)d] - [%(levelname)s] - %(message)s')
# 创建一个handler，用于写入日志文件
filepath = BASEDIR+'/output/log/' + genFilename(self.targetname) + '.log'
if os.path.isfile(filepath):
os.remove(filepath)
fh = logging.FileHandler(filepath,'a')
# 再创建一个handler，用于输出到控制台
ch = logging.StreamHandler()
fi = logging.Filter('main')
fh.addFilter(fi)
ch.addFilter(fi)
fh.setFormatter(formatter)
ch.setFormatter(formatter)
self.loghandler.append(ch)
self.loghandler.append(fh)
self._initLogging()
globalVar.mainlogger.info('[*] Start a new scan')
globalVar.mainlogger.info('\tserver\t=%s' % self.server)
globalVar.mainlogger.info('\ttoken\t=%s' % self.token)
globalVar.mainlogger.info('\ttarget\t=%s' % self.target)
globalVar.mainlogger.info('\tthreads\t=%d' % self.threads)
# 注意：不能通过以下的方式进行清空
# globalVar.undone_targets = []
tmpundone = copy.deepcopy(globalVar.undone_targets)
for each_target in tmpundone:
globalVar.undone_targets.remove(each_target)
def _initLogging(self):
# globalVar.mainlogger.info('before test')
for handler in self.loghandler:
globalVar.mainlogger.addHandler(handler)
# globalVar.mainlogger.info('after test')
def _removeLogging(self):
# globalVar.mainlogger.info('before test')
for handler in self.loghandler:
globalVar.mainlogger.removeHandler(handler)
# globalVar.mainlogger.info('after test')
globalVar.mainlogger = None
def _getServiceType(self,target):
m = re.search('(http[s]?)://([^:^/]+):?([^/]*)/?',target)
if m:
return 'url'
else:
m = re.search('^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$',target)
if m:
return 'ip'
else:
return 'host'
def _noticeStartToWeb(self):
''' '''
# print '>>notice server start scan'
globalVar.mainlogger.info('Notice server start scan')
if self.web_interface == None:
# print'server not exists'
globalVar.mainlogger.error('\tserver not exists')
return False
# save Scan table at first
# print 'self.targetname\t',self.targetname
self.web_interface.task_start(self.targetname,str(self.args))
def _initGlobalVar(self):
# process information
# print 'in scaner_class_mp process pid=\t',os.getpid()
# print 'id(globalVar)=\t',id(globalVar)
# print globals()
pid = os.getpid()
globalVar.scan_task_dict_lock.acquire()
globalVar.scan_task_dict['pid'] = pid
globalVar.scan_task_dict['target'] = self.target
globalVar.scan_task_dict['targetname'] = self.targetname
globalVar.scan_task_dict['server'] = self.web_interface.server
globalVar.scan_task_dict['token'] = self.web_interface.token
globalVar.scan_task_dict['subtargets'] = {}
globalVar.scan_task_dict['scanID'] = self.web_interface.id
globalVar.scan_task_dict_lock.release()
def _saveResultToWeb(self):
# print '>>>saving scan result to server'
globalVar.mainlogger.info('Saving scan result to server')
if self.web_interface == None:
globalVar.mainlogger.error('\tserver not exists')
# print'server not exists'
return False
else:
self.web_interface.task_end()
def initInfo(self,target=None):
try:
# Step 1
globalVar.mainlogger.info('[*][*] Step1: init starting info')
self.services = []
if target==None:
target = self.target
targets = []
for each_target in target.split('\n'):
# if each_target:
# m = re.search('^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/\d{1,2}$',each_target)
# if m:
# ipnet = list(netaddr.IPNetwork(each_target))
# for eachip in ipnet:
# targets.append(eachip.format())
# elif re.match('^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$',each_target) \
# or re.match('(http[s]?)://([^:^/]+):?([^/]*)/?',each_target) \
# or re.match('(?i)^([a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,}$',each_target):
# targets.append(each_target)
if each_target:
# ip range type
m = re.search('^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/\d{1,2}$',each_target)
if m:
ipnet = list(netaddr.IPNetwork(each_target))
for eachip in ipnet:
targets.append(eachip.format())
else:
# one ip
m = re.search('^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$',each_target)
if m:
targets.append(each_target)
else:
# url type
m = re.match('(http[s]?)://([^:^/]+):?([^/]*)/?',each_target)
if m:
http_type = m.group(1)
# print m.group(2)
n = re.search('^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}(\/\d{1,2})?$',m.group(2))
# ip
if n:
# print 'is an ip type url'
ip = m.group(2)
if each_target[-1] == '/':
each_target = each_target[:-1]
targets.append(ip)
targets.append(each_target)
else:
host = m.group(2)
ports = m.group(3)
# print host
ip = socket.gethostbyname(host)
domain = GetFirstLevelDomain(host)
# print 'ip=',ip
if each_target[-1] == '/':
each_target = each_target[:-1]
targets.append(ip)
targets.append(each_target)
targets.append(domain)
else:
# host type
domain = GetFirstLevelDomain(each_target)
targets.append(domain)
# 去重
targets = list(set(targets))
# for each_target in globalVar.undone_targets:
for each_target in targets:
if each_target:
globalVar.undone_targets.append(each_target)
service = {}
# print each_target
service_type = self._getServiceType(each_target)
service[service_type] = each_target
self.services.append(service)
print 'globalVar.undone_targets=',globalVar.undone_targets
print 'self.services=',
pprint(self.services)
globalVar.mainlogger.info('Targets:')
for service in self.services:
globalVar.mainlogger.info('\t'+str(service))
self._noticeStartToWeb()
self._initGlobalVar()
except IndexError,e:
# except Exception,e:
globalVar.mainlogger.error('Exception:'+str(e))
def __init__(self,
server=None,
token=None,
target=None,
threads=None,
loglevel='INFO',
gatherdepth=1):
super(Scanner, self).__init__()
self.server = server
self.token = token
self.target = target
if threads and type(threads) == int:
self.threads = threads
else:
self.threads = multiprocessing.cpu_count()
self.gatherdepth = gatherdepth
self.loglevel = loglevel
self.args = {
'loglevel': self.loglevel,
'threads': self.threads,
'gatherdepth': self.gatherdepth
}
# web接口
self.web_interface = None
if server and token:
self.web_interface = WebInterface(server, token)
# 任务
self.services = []
# 扫描结果
self.result = {}
# pluginLoaders
self.pls = []
self.loghandler = []
# log 模块,确保赋值一次
if globalVar.mainlogger is None:
globalVar.mainlogger = logging.getLogger('main')
if loglevel == 'DEBUG':
globalVar.mainlogger.setLevel(logging.DEBUG)
else:
globalVar.mainlogger.setLevel(logging.INFO)
# logging handler
formatter = logging.Formatter(
'[%(process)d] - [%(levelname)s] - %(message)s')
# 创建一个handler，用于写入日志文件
filepath = BASEDIR + '/output/log/' + genFilename(
self.target) + '.log'
if os.path.isfile(filepath):
os.remove(filepath)
fh = logging.FileHandler(filepath, 'a')
# 再创建一个handler，用于输出到控制台
ch = logging.StreamHandler()
fi = logging.Filter('main')
fh.addFilter(fi)
ch.addFilter(fi)
fh.setFormatter(formatter)
ch.setFormatter(formatter)
self.loghandler.append(ch)
self.loghandler.append(fh)
self._initLogging()
globalVar.mainlogger.info('[*] Start a new scan')
globalVar.mainlogger.info('\tserver\t=%s' % server)
globalVar.mainlogger.info('\ttoken\t=%s' % token)
globalVar.mainlogger.info('\ttarget\t=%s' % target)
globalVar.mainlogger.info('\tthreads\t=%d' % self.threads)
# 注意：不能通过以下的方式进行清空
# globalVar.undone_targets = []
tmpundone = copy.deepcopy(globalVar.undone_targets)
for each_target in tmpundone:
globalVar.undone_targets.remove(each_target)
class Scanner(object):
"""docstring for Scanner"""
def __init__(self,
server=None,
token=None,
target=None,
threads=None,
loglevel='INFO',
gatherdepth=1):
super(Scanner, self).__init__()
self.server = server
self.token = token
self.target = target
if threads and type(threads) == int:
self.threads = threads
else:
self.threads = multiprocessing.cpu_count()
self.gatherdepth = gatherdepth
self.loglevel = loglevel
self.args = {
'loglevel': self.loglevel,
'threads': self.threads,
'gatherdepth': self.gatherdepth
}
# web接口
self.web_interface = None
if server and token:
self.web_interface = WebInterface(server, token)
# 任务
self.services = []
# 扫描结果
self.result = {}
# pluginLoaders
self.pls = []
self.loghandler = []
# log 模块,确保赋值一次
if globalVar.mainlogger is None:
globalVar.mainlogger = logging.getLogger('main')
if loglevel == 'DEBUG':
globalVar.mainlogger.setLevel(logging.DEBUG)
else:
globalVar.mainlogger.setLevel(logging.INFO)
# logging handler
formatter = logging.Formatter(
'[%(process)d] - [%(levelname)s] - %(message)s')
# 创建一个handler，用于写入日志文件
filepath = BASEDIR + '/output/log/' + genFilename(
self.target) + '.log'
if os.path.isfile(filepath):
os.remove(filepath)
fh = logging.FileHandler(filepath, 'a')
# 再创建一个handler，用于输出到控制台
ch = logging.StreamHandler()
fi = logging.Filter('main')
fh.addFilter(fi)
ch.addFilter(fi)
fh.setFormatter(formatter)
ch.setFormatter(formatter)
self.loghandler.append(ch)
self.loghandler.append(fh)
self._initLogging()
globalVar.mainlogger.info('[*] Start a new scan')
globalVar.mainlogger.info('\tserver\t=%s' % server)
globalVar.mainlogger.info('\ttoken\t=%s' % token)
globalVar.mainlogger.info('\ttarget\t=%s' % target)
globalVar.mainlogger.info('\tthreads\t=%d' % self.threads)
# 注意：不能通过以下的方式进行清空
# globalVar.undone_targets = []
tmpundone = copy.deepcopy(globalVar.undone_targets)
for each_target in tmpundone:
globalVar.undone_targets.remove(each_target)
def _initLogging(self):
# globalVar.mainlogger.info('before test')
for handler in self.loghandler:
globalVar.mainlogger.addHandler(handler)
# globalVar.mainlogger.info('after test')
def _removeLogging(self):
# globalVar.mainlogger.info('before test')
for handler in self.loghandler:
globalVar.mainlogger.removeHandler(handler)
# globalVar.mainlogger.info('after test')
globalVar.mainlogger = None
def _getServiceType(self, target):
m = re.search('(http[s]?)://([^:^/]+):?([^/]*)/?', target)
if m:
return 'url'
else:
m = re.search('^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$', target)
if m:
return 'ip'
else:
return 'host'
def _noticeStartToWeb(self):
''' '''
# print '>>notice server start scan'
globalVar.mainlogger.info('Notice server start scan')
if self.web_interface == None:
# print'server not exists'
globalVar.mainlogger.error('\tserver not exists')
return False
# save Scan table at first
# print 'self.target\t',self.target
self.web_interface.task_start(self.target, str(self.args))
def _initGlobalVar(self):
# process information
# print 'in scaner_class_mp process pid=\t',os.getpid()
# print 'id(globalVar)=\t',id(globalVar)
# print globals()
pid = os.getpid()
globalVar.scan_task_dict_lock.acquire()
globalVar.scan_task_dict['pid'] = pid
globalVar.scan_task_dict['target'] = self.target
globalVar.scan_task_dict['server'] = self.web_interface.server
globalVar.scan_task_dict['token'] = self.web_interface.token
globalVar.scan_task_dict['subtargets'] = {}
globalVar.scan_task_dict['scanID'] = self.web_interface.id
globalVar.scan_task_dict_lock.release()
def _saveResultToWeb(self):
# print '>>>saving scan result to server'
globalVar.mainlogger.info('Saving scan result to server')
if self.web_interface == None:
globalVar.mainlogger.error('\tserver not exists')
# print'server not exists'
return False
else:
self.web_interface.task_end()
def initInfo(self, target=None):
try:
# Step 1
globalVar.mainlogger.info('[*][*] Step1: init starting info')
self.services = []
if target == None:
target = self.target
targets = []
if target:
# ip range type
m = re.search('^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/\d{1,2}$',
target)
if m:
ipnet = list(netaddr.IPNetwork(target))
for eachip in ipnet:
targets.append(eachip.format())
# ipnet = list(ipaddress.ip_network(unicode(target)).hosts())
# for eachipad in ipnet:
# targets.append(eachipad.compressed)
else:
# one ip
m = re.search('^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$',
target)
if m:
targets.append(target)
else:
# url type
m = re.match('(http[s]?)://([^:^/]+):?([^/]*)/?',
target)
if m:
http_type = m.group(1)
# print m.group(2)
n = re.search(
'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}(\/\d{1,2})?$',
m.group(2))
# ip
if n:
# print 'is an ip type url'
ip = m.group(2)
if target[-1] == '/':
target = target[:-1]
targets.append(ip)
targets.append(target)
else:
host = m.group(2)
ports = m.group(3)
# print host
ip = socket.gethostbyname(host)
domain = GetFirstLevelDomain(host)
# print 'ip=',ip
if target[-1] == '/':
target = target[:-1]
targets.append(ip)
targets.append(target)
targets.append(domain)
else:
# host type
domain = GetFirstLevelDomain(target)
targets.append(domain)
# for each_target in globalVar.undone_targets:
for each_target in targets:
globalVar.undone_targets.append(each_target)
service = {}
service_type = self._getServiceType(each_target)
# print service_type
service[service_type] = each_target
self.services.append(service)
print 'globalVar.undone_targets=', globalVar.undone_targets
print 'self.services=',
pprint(self.services)
globalVar.mainlogger.info('Targets:')
for service in self.services:
globalVar.mainlogger.info('\t' + str(service))
self._noticeStartToWeb()
self._initGlobalVar()
except IndexError, e:
# except Exception,e:
globalVar.mainlogger.error('Exception:' + str(e))
def __init__(self,server=None,token=None,target=None,threads=None,loglevel='INFO',gatherdepth=1):
super(Scanner, self).__init__()
self.server = server
self.token = token
self.target = target
if threads and type(threads) == int:
self.threads = threads
else:
self.threads = multiprocessing.cpu_count()
self.gatherdepth = gatherdepth
self.loglevel = loglevel
self.args = {'loglevel':self.loglevel,'threads':self.threads,'gatherdepth':self.gatherdepth}
# web接口
self.web_interface = None
if server and token:
self.web_interface = WebInterface(server,token)
# 任务
self.services = []
# 扫描结果
self.result = {}
# pluginLoaders
self.pls = []
self.loghandler = []
# log 模块,确保赋值一次
if globalVar.mainlogger is None:
globalVar.mainlogger = logging.getLogger('main')
if loglevel == 'DEBUG':
globalVar.mainlogger.setLevel(logging.DEBUG)
else:
globalVar.mainlogger.setLevel(logging.INFO)
# logging handler
formatter = logging.Formatter('[%(process)d] - [%(levelname)s] - %(message)s')
# 创建一个handler，用于写入日志文件
filepath = BASEDIR+'/output/log/' + genFilename(self.target) + '.log'
if os.path.isfile(filepath):
os.remove(filepath)
fh = logging.FileHandler(filepath,'a')
# 再创建一个handler，用于输出到控制台
ch = logging.StreamHandler()
fi = logging.Filter('main')
fh.addFilter(fi)
ch.addFilter(fi)
fh.setFormatter(formatter)
ch.setFormatter(formatter)
self.loghandler.append(ch)
self.loghandler.append(fh)
self._initLogging()
globalVar.mainlogger.info('[*] Start a new scan')
globalVar.mainlogger.info('\tserver\t=%s' % server)
globalVar.mainlogger.info('\ttoken\t=%s' % token)
globalVar.mainlogger.info('\ttarget\t=%s' % target)
globalVar.mainlogger.info('\tthreads\t=%d' % self.threads)
# 注意：不能通过以下的方式进行清空
# globalVar.undone_targets = []
tmpundone = copy.deepcopy(globalVar.undone_targets)
for each_target in tmpundone:
globalVar.undone_targets.remove(each_target)
def __init__(self,server=None,token=None,target=None,pluginfilepath=None,pluginargs=None,threads=None,loglevel='INFO'):
super(PluginMultiRunner, self).__init__()
self.server = server
self.token = token
self.target = target
self.pluginfilepath = BASEDIR +'/' +pluginfilepath
self.pluginargs = pluginargs
if threads and type(threads) == int:
self.threads = int(threads)
else:
self.threads = multiprocessing.cpu_count()
self.loglevel = loglevel
self.args = {'loglevel':self.loglevel,'threads':self.threads}
# web接口
self.web_interface = None
if server and token:
self.web_interface = WebInterface(server,token)
# 任务
self.services = []
# 扫描结果
self.result = {}
# pluginLoaders
self.pls = []
# log 模块
globalVar.mainlogger = logging.getLogger('main')
if loglevel == 'DEBUG':
globalVar.mainlogger.setLevel(logging.DEBUG)
else:
globalVar.mainlogger.setLevel(logging.INFO)
# 定义handler的输出格式formatter
# formatter = logging.Formatter('%(asctime)s - %(name)s - [%(levelname)s] - %(message)s')
formatter1 = logging.Formatter('[%(process)d] - [%(levelname)s] - %(message)s')
formatter2 = logging.Formatter('%(message)s')
# 创建一个handler，用于写入日志文件
filepath = BASEDIR+'/output/scan.log'
if os.path.isfile(filepath):
os.remove(filepath)
fh = logging.FileHandler(filepath,'a')
# 再创建一个handler，用于输出到控制台
ch = logging.StreamHandler()
fi = logging.Filter('main')
fh.addFilter(fi)
ch.addFilter(fi)
fh.setFormatter(formatter1)
ch.setFormatter(formatter1)
globalVar.mainlogger.addHandler(fh)
globalVar.mainlogger.addHandler(ch)
globalVar.mainlogger.info('[*] Start an new scan')
globalVar.mainlogger.info('\tserver =%s' % server)
globalVar.mainlogger.info('\ttoken =%s' % token)
globalVar.mainlogger.info('\ttarget =%s' % target)
globalVar.mainlogger.info('\tthreads =%d' % self.threads)
class PluginMultiRunner(object):
"""docstring for Scanner"""
def __init__(self,server=None,token=None,target=None,pluginfilepath=None,pluginargs=None,threads=None,loglevel='INFO'):
super(PluginMultiRunner, self).__init__()
self.server = server
self.token = token
self.target = target
self.pluginfilepath = BASEDIR +'/' +pluginfilepath
self.pluginargs = pluginargs
if threads and type(threads) == int:
self.threads = int(threads)
else:
self.threads = multiprocessing.cpu_count()
self.loglevel = loglevel
self.args = {'loglevel':self.loglevel,'threads':self.threads}
# web接口
self.web_interface = None
if server and token:
self.web_interface = WebInterface(server,token)
# 任务
self.services = []
# 扫描结果
self.result = {}
# pluginLoaders
self.pls = []
# log 模块
globalVar.mainlogger = logging.getLogger('main')
if loglevel == 'DEBUG':
globalVar.mainlogger.setLevel(logging.DEBUG)
else:
globalVar.mainlogger.setLevel(logging.INFO)
# 定义handler的输出格式formatter
# formatter = logging.Formatter('%(asctime)s - %(name)s - [%(levelname)s] - %(message)s')
formatter1 = logging.Formatter('[%(process)d] - [%(levelname)s] - %(message)s')
formatter2 = logging.Formatter('%(message)s')
# 创建一个handler，用于写入日志文件
filepath = BASEDIR+'/output/scan.log'
if os.path.isfile(filepath):
os.remove(filepath)
fh = logging.FileHandler(filepath,'a')
# 再创建一个handler，用于输出到控制台
ch = logging.StreamHandler()
fi = logging.Filter('main')
fh.addFilter(fi)
ch.addFilter(fi)
fh.setFormatter(formatter1)
ch.setFormatter(formatter1)
globalVar.mainlogger.addHandler(fh)
globalVar.mainlogger.addHandler(ch)
globalVar.mainlogger.info('[*] Start an new scan')
globalVar.mainlogger.info('\tserver =%s' % server)
globalVar.mainlogger.info('\ttoken =%s' % token)
globalVar.mainlogger.info('\ttarget =%s' % target)
globalVar.mainlogger.info('\tthreads =%d' % self.threads)
def _getServiceType(self,target):
m = re.search('(http[s]?)://([^:^/]+):?([^/]*)/?',target)
if m:
return 'url'
else:
m = re.search('^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$',target)
if m:
return 'ip'
else:
return 'host'
def _noticeStartToWeb(self):
''' '''
# print '>>notice server start scan'
globalVar.mainlogger.info('Notice server start scan')
if self.web_interface == None:
# print'server not exists'
globalVar.mainlogger.error('\tserver not exists')
return False
# save Scan table at first
# print 'self.target\t',self.target
self.web_interface.task_start(self.target,str(self.args))
def _initGlobalVar(self):
# process information
# print 'in scaner_class_mp process pid=\t',os.getpid()
# print 'id(globalVar)=\t',id(globalVar)
# print globals()
pid = os.getpid()
globalVar.scan_task_dict_lock.acquire()
globalVar.scan_task_dict['pid'] = pid
globalVar.scan_task_dict['target'] = self.target
globalVar.scan_task_dict['server'] = self.web_interface.server
globalVar.scan_task_dict['token'] = self.web_interface.token
globalVar.scan_task_dict['subtargets'] = {}
globalVar.scan_task_dict['scanID'] = self.web_interface.id
globalVar.scan_task_dict_lock.release()
def _saveResultToWeb(self):
globalVar.mainlogger.info('Saving scan result to server')
if self.web_interface == None:
globalVar.mainlogger.error('\tserver not exists')
return False
else:
self.web_interface.task_end()
def initInfo(self,target=None):
try:
# Step 1
globalVar.mainlogger.info('[*][*] Step1: init starting info')
if target==None:
target = self.target
targets = []
# file type target
if os.path.isfile(target):
for eachLine in f:
eachLine = eachLine.replace('\r','')
eachLine = eachLine.replace('\n','')
targets.append(eachLine)
# ip range type
else:
m = re.search('^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}(\/\d{1,2})?$',target)
if m:
# 弃用ipaddress库，因为ip range解析问题
#
# ipnet = list(ipaddress.ip_network(unicode(target)).hosts())
# for eachipad in ipnet:
# targets.append(eachipad.compressed)
ipnet = list(netaddr.IPNetwork(target))
for eachip in ipnet:
targets.append(eachip.format())
else:
targets.append(target)
argdict = {}
if self.pluginargs:
pluginargs = self.pluginargs.split(';')
for eacharg in pluginargs:
if '=' in eacharg:
exec(eacharg)
eacharg = eacharg.split('=')
argdict[eacharg[0]] = eval(eacharg[0])
globalVar.mainlogger.debug('argdict=%s' % str(argdict))
for each_target in targets:
service = {'nogather':'True'}
service_type = self._getServiceType(each_target)
service[service_type] = each_target
if len(argdict):
service.update(argdict)
self.services.append(service)
globalVar.mainlogger.info('Targets:')
for service in self.services:
globalVar.mainlogger.info('\t'+str(service))
self._noticeStartToWeb()
self._initGlobalVar()
except IndexError,e:
# except Exception,e:
globalVar.mainlogger.error('Exception:'+str(e))
def __init__(self, conffile):
super(Scanner, self).__init__()
config = json.load(open(conffile, 'r'))
# 1. init globalVar.config first
globalVar.config = config
pprint(globalVar.config['global'])
# 2.
self.server = config['global']['server']
self.token = config['global']['token']
# 注意targetname直接在config的key,而不是config['global']的key
self.targetname = config['targetname']
self.target = config['global']['target']
self.threads = int(
config['global']['threads']
) if config['global']['threads'] != '' else multiprocessing.cpu_count(
)
print 'self.threads=', self.threads, type(self.threads)
# print "config['global']['gatherdepth']=",config['global']['gatherdepth']
self.gatherdepth = int(
config['global']
['gatherdepth']) if config['global']['gatherdepth'] != '' else 0
# print 'self.gatherdepth=',self.gatherdepth
self.loglevel = config['global']['loglevel'] if config['global'][
'threads'] == '' else 'INFO'
self.args = {
'loglevel': self.loglevel,
'threads': self.threads,
'gatherdepth': self.gatherdepth
}
self.pluginargs = config['plugins']
# web接口
self.web_interface = None
if self.server and self.token:
self.web_interface = WebInterface(self.server, self.token)
# 任务
self.services = []
# 扫描结果
self.result = {}
# pluginLoaders
self.pls = []
# 3. init logging
self.loghandler = []
# log 模块,确保赋值一次
if globalVar.mainlogger is None:
globalVar.mainlogger = logging.getLogger('main')
if self.loglevel == 'DEBUG':
globalVar.mainlogger.setLevel(logging.DEBUG)
else:
globalVar.mainlogger.setLevel(logging.INFO)
# logging handler
formatter = logging.Formatter(
'[%(process)d] - [%(levelname)s] - %(message)s')
# 创建一个handler，用于写入日志文件
filepath = BASEDIR + '/output/log/' + genFilename(
self.targetname) + '.log'
if os.path.isfile(filepath):
os.remove(filepath)
fh = logging.FileHandler(filepath, 'a')
# 再创建一个handler，用于输出到控制台
ch = logging.StreamHandler()
fi = logging.Filter('main')
fh.addFilter(fi)
ch.addFilter(fi)
fh.setFormatter(formatter)
ch.setFormatter(formatter)
self.loghandler.append(ch)
self.loghandler.append(fh)
self._initLogging()
globalVar.mainlogger.info('[*] Start a new scan')
globalVar.mainlogger.info('\tserver\t=%s' % self.server)
globalVar.mainlogger.info('\ttoken\t=%s' % self.token)
globalVar.mainlogger.info('\ttarget\t=%s' % self.target)
globalVar.mainlogger.info('\tthreads\t=%d' % self.threads)
# 注意：不能通过以下的方式进行清空
# globalVar.undone_targets = []
tmpundone = copy.deepcopy(globalVar.undone_targets)
for each_target in tmpundone:
globalVar.undone_targets.remove(each_target)
class Scanner(object):
"""docstring for Scanner"""
def __init__(self,server=None,token=None,target=None):
super(Scanner, self).__init__()
self.server = server
self.token = token
self.target = target
# web接口
self.web_interface = None
if server and token:
self.web_interface = WebInterface(server,token)
# 任务
self.services = []
# 扫描结果
self.result = {}
# pluginLoaders
self.pls = []
def _getServiceType(self,target):
m = re.search('(http[s]?)://([^:^/]+):?([^/]*)/?',target)
if m:
return 'url'
else:
m = re.search('^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$',target)
if m:
return 'ip'
else:
return 'host'
def _noticeStartToWeb(self):
''' '''
print '>>notice server start scan'
if self.web_interface == None:
print'server not exists'
return False
# save Scan table at first
print 'self.target\t',self.target
self.web_interface.task_start(self.target,self.target)
def _initGlobalVar(self):
# process information
# print 'in scaner_class_mp process pid=\t',os.getpid()
# print 'id(globalVar)=\t',id(globalVar)
# print globals()
pid = os.getpid()
globalVar.scan_task_dict_lock.acquire()
globalVar.scan_task_dict['pid'] = pid
globalVar.scan_task_dict['target'] = self.target
globalVar.scan_task_dict['server'] = self.web_interface.server
globalVar.scan_task_dict['token'] = self.web_interface.token
globalVar.scan_task_dict['subtargets'] = {}
globalVar.scan_task_dict['scanID'] = self.web_interface.id
globalVar.scan_task_dict_lock.release()
def _saveResultToWeb(self):
print '>>>saving scan result to server'
if self.web_interface == None:
print'server not exists'
return False
else:
self.web_interface.task_end()
def _saveResultToFile(self,pls=None,outputpath=None):
''' 这个函数已经弃用，在PluginLoader运行过程中，每运行一个插件都会实时将结果写入，所有不需要在扫描完成后再次写入'''
print '>>>saving scan result to file'
if pls == None:
pls = self.pls
if outputpath == None:
target = self.target.replace('://','_')
target = target.replace('/','')
outputpath = BASEDIR + '/output/' + target
if os.path.isdir(outputpath) == False:
os.makedirs(outputpath)
for eachpl in pls:
tmp =''
if eachpl.services.has_key('ip'):
threadName = eachpl.services['ip']
eachfile = outputpath + '/' + threadName
tmp += '*'*25 + ' scan info '+ '*'*25 + os.linesep
tmp += '# this is an ip type scan' + os.linesep
tmp += 'ip:\t' + threadName + os.linesep
elif eachpl.services.has_key('url'):
threadName = eachpl.services['url']
tmpurl = threadName.replace('://','_')
tmpurl = tmpurl.replace(':','_')
tmpurl = tmpurl.replace('/','')
eachfile = outputpath + '/' + tmpurl
tmp += '*'*25 + ' scan info '+ '*'*25 + os.linesep
tmp += '# this is a http type scan' + os.linesep
tmp += 'url:\t' + threadName + os.linesep
elif eachpl.services.has_key('host'):
threadName = eachpl.services['host']
eachfile = outputpath + '/' + threadName
tmp += '*'*25 + ' scan info '+ '*'*25 + os.linesep
tmp += '# this is an ip type scan' + os.linesep
tmp += 'host:\t' + threadName + os.linesep
tmp += '*'*25 + ' scan output '+ '*'*25 + os.linesep
tmp += eachpl.output + os.linesep
tmp += '*'*25 + ' scan services '+ '*'*25 + os.linesep
tmp += str(eachpl.services) + os.linesep
tmp += '*'*25 + ' scan result '+ '*'*25 + os.linesep
tmp += str(eachpl.retinfo) + os.linesep
fp = open(eachfile,'w')
fp.write(tmp)
fp.close()
def initInfo(self,target=None):
try:
# Step 1
print '>>>Step1: init starting info'
if target==None:
target = self.target
targets = []
if target:
# ip range type
m = re.search('^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/\d{1,2}$',target)
if m:
ipnet = list(ipaddress.ip_network(unicode(target)).hosts())
for eachipad in ipnet:
targets.append(eachipad.compressed)
else:
# one ip
m = re.search('^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$',target)
if m:
targets.append(target)
else:
# url type
m = re.match('(http[s]?)://([^:^/]+):?([^/]*)/?',target)
if m:
http_type = m.group(1)
host = m.group(2)
ports = m.group(3)
ip = socket.gethostbyname(host)
# domain = GetFirstLevelDomain(host)
# print 'ip=',ip
targets.append(ip)
targets.append(target)
targets.append(host)
else:
# host type
pass
globalVar.target_lock.acquire()
globalVar.undone_targets += targets
globalVar.target_lock.release()
self._noticeStartToWeb()
self._initGlobalVar()
except Exception,e:
print 'Exception',e
def __init__(self,server=None,token=None,target=None,pluginfilepath=None,pluginargs=None,threads=None,loglevel='INFO'):
super(PluginMultiRunner, self).__init__()
self.server = server
self.token = token
self.target = target
self.pluginfilepath = BASEDIR +'/' +pluginfilepath
self.pluginargs = pluginargs
if threads and type(threads) == int:
self.threads = int(threads)
else:
self.threads = multiprocessing.cpu_count()
self.loglevel = loglevel
self.args = {'loglevel':self.loglevel,'threads':self.threads}
# web接口
self.web_interface = None
if server and token:
self.web_interface = WebInterface(server,token)
# 任务
self.services = []
# 扫描结果
self.result = {}
# pluginLoaders
self.pls = []
# log 模块
globalVar.mainlogger = logging.getLogger('main')
if loglevel == 'DEBUG':
globalVar.mainlogger.setLevel(logging.DEBUG)
else:
globalVar.mainlogger.setLevel(logging.INFO)
# 定义handler的输出格式formatter
# formatter = logging.Formatter('%(asctime)s - %(name)s - [%(levelname)s] - %(message)s')
formatter1 = logging.Formatter('[%(process)d] - [%(levelname)s] - %(message)s')
formatter2 = logging.Formatter('%(message)s')
# 创建一个handler，用于写入日志文件
filepath = BASEDIR+'/output/scan.log'
if os.path.isfile(filepath):
os.remove(filepath)
fh = logging.FileHandler(filepath,'a')
# 再创建一个handler，用于输出到控制台
ch = logging.StreamHandler()
fi = logging.Filter('main')
fh.addFilter(fi)
ch.addFilter(fi)
fh.setFormatter(formatter1)
ch.setFormatter(formatter1)
globalVar.mainlogger.addHandler(fh)
globalVar.mainlogger.addHandler(ch)
globalVar.mainlogger.info('[*] Start an new scan')
globalVar.mainlogger.info('\tserver =%s' % server)
globalVar.mainlogger.info('\ttoken =%s' % token)
globalVar.mainlogger.info('\ttarget =%s' % target)
globalVar.mainlogger.info('\tthreads =%d' % self.threads)
class PluginMultiRunner(object):
"""docstring for Scanner"""
def __init__(self,server=None,token=None,target=None,pluginfilepath=None,pluginargs=None,threads=None,loglevel='INFO'):
super(PluginMultiRunner, self).__init__()
self.server = server
self.token = token
self.target = target
self.pluginfilepath = BASEDIR +'/' +pluginfilepath
self.pluginargs = pluginargs
if threads and type(threads) == int:
self.threads = int(threads)
else:
self.threads = multiprocessing.cpu_count()
self.loglevel = loglevel
self.args = {'loglevel':self.loglevel,'threads':self.threads}
# web接口
self.web_interface = None
if server and token:
self.web_interface = WebInterface(server,token)
# 任务
self.services = []
# 扫描结果
self.result = {}
# pluginLoaders
self.pls = []
# log 模块
globalVar.mainlogger = logging.getLogger('main')
if loglevel == 'DEBUG':
globalVar.mainlogger.setLevel(logging.DEBUG)
else:
globalVar.mainlogger.setLevel(logging.INFO)
# 定义handler的输出格式formatter
# formatter = logging.Formatter('%(asctime)s - %(name)s - [%(levelname)s] - %(message)s')
formatter1 = logging.Formatter('[%(process)d] - [%(levelname)s] - %(message)s')
formatter2 = logging.Formatter('%(message)s')
# 创建一个handler，用于写入日志文件
filepath = BASEDIR+'/output/scan.log'
if os.path.isfile(filepath):
os.remove(filepath)
fh = logging.FileHandler(filepath,'a')
# 再创建一个handler，用于输出到控制台
ch = logging.StreamHandler()
fi = logging.Filter('main')
fh.addFilter(fi)
ch.addFilter(fi)
fh.setFormatter(formatter1)
ch.setFormatter(formatter1)
globalVar.mainlogger.addHandler(fh)
globalVar.mainlogger.addHandler(ch)
globalVar.mainlogger.info('[*] Start an new scan')
globalVar.mainlogger.info('\tserver =%s' % server)
globalVar.mainlogger.info('\ttoken =%s' % token)
globalVar.mainlogger.info('\ttarget =%s' % target)
globalVar.mainlogger.info('\tthreads =%d' % self.threads)
def _getServiceType(self,target):
m = re.search('(http[s]?)://([^:^/]+):?([^/]*)/?',target)
if m:
return 'url'
else:
m = re.search('^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$',target)
if m:
return 'ip'
else:
return 'host'
def _noticeStartToWeb(self):
''' '''
# print '>>notice server start scan'
globalVar.mainlogger.info('Notice server start scan')
if self.web_interface == None:
# print'server not exists'
globalVar.mainlogger.error('\tserver not exists')
return False
# save Scan table at first
# print 'self.target\t',self.target
self.web_interface.task_start(self.target,str(self.args))
def _initGlobalVar(self):
# process information
# print 'in scaner_class_mp process pid=\t',os.getpid()
# print 'id(globalVar)=\t',id(globalVar)
# print globals()
pid = os.getpid()
globalVar.scan_task_dict_lock.acquire()
globalVar.scan_task_dict['pid'] = pid
globalVar.scan_task_dict['target'] = self.target
globalVar.scan_task_dict['server'] = self.web_interface.server
globalVar.scan_task_dict['token'] = self.web_interface.token
globalVar.scan_task_dict['subtargets'] = {}
globalVar.scan_task_dict['scanID'] = self.web_interface.id
globalVar.scan_task_dict_lock.release()
def _saveResultToWeb(self):
globalVar.mainlogger.info('Saving scan result to server')
if self.web_interface == None:
globalVar.mainlogger.error('\tserver not exists')
return False
else:
self.web_interface.task_end()
def initInfo(self,target=None):
try:
# Step 1
globalVar.mainlogger.info('[*][*] Step1: init starting info')
if target==None:
target = self.target
targets = []
# file type target
if os.path.isfile(target):
f= open(target)
for eachLine in f:
eachLine = eachLine.replace('\r','')
eachLine = eachLine.replace('\n','')
targets.append(eachLine)
# ip range type
else:
m = re.search('^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}(\/\d{1,2})?$',target)
if m:
# 弃用ipaddress库，因为ip range解析问题
#
# ipnet = list(ipaddress.ip_network(unicode(target)).hosts())
# for eachipad in ipnet:
# targets.append(eachipad.compressed)
ipnet = list(netaddr.IPNetwork(target))
for eachip in ipnet:
targets.append(eachip.format())
else:
targets.append(target)
argdict = {}
if self.pluginargs:
pluginargs = self.pluginargs.split(';')
for eacharg in pluginargs:
if '=' in eacharg:
exec(eacharg)
eacharg = eacharg.split('=')
argdict[eacharg[0]] = eval(eacharg[0])
globalVar.mainlogger.debug('argdict=%s' % str(argdict))
for each_target in targets:
service = {'nogather':'True'}
service_type = self._getServiceType(each_target)
service[service_type] = each_target
if len(argdict):
service.update(argdict)
self.services.append(service)
globalVar.mainlogger.info('Targets:')
for service in self.services:
globalVar.mainlogger.info('\t'+str(service))
self._noticeStartToWeb()
self._initGlobalVar()
except IndexError,e:
# except Exception,e:
globalVar.mainlogger.error('Exception:'+str(e))
