def guideRegister(args):
#向导模式
#有url 没选插件参数时
if args.u and not args.plugin:
inputUrl = args.u
urlconfig.url.append(makeurl(inputUrl))
printMessage('[Prompt] URL has been loaded:%d' % len(urlconfig.url))
urlconfig.diyPlugin = ["find_service", "whatcms"] #插件选择
printMessage("[Prompt] You select the plugins:%s" %
(' '.join(urlconfig.diyPlugin)))
urlconfig.scanport = False #端口扫描默认关闭
urlconfig.find_service = True #服务信息扫描默认开启
return True
#有url 有选插件参数时
if args.u and args.plugin:
return False
#无url 和插件参数时
inputUrl = input('[1] Input url > ')
if inputUrl == '':
raise ToolkitSystemException("You have to enter the url")
#输入为文件时:
if inputUrl.startswith("@"):
urlconfig.mutiurl = True
filename = inputUrl[1:]
try:
o = open(filename, "r").readlines() #一行行读取
for url in o:
urlconfig.url.append(makeurl(url.strip()))
except IOError:
raise ToolkitSystemException("Filename:'%s' open faild" % fileName)
if len(o) == 0:
raise ToolkitSystemException("The target address is empty")
else:
urlconfig.url.append(makeurl(inputUrl))
printMessage('[Prompt] URL has been loaded:%d' % len(urlconfig.url))
printMessage("[Prompt] You can select these plugins (%s) or select all" %
(' '.join(LIST_PLUGINS)))
diyPlugin = input("[2] Please select the required plugins > ")
if diyPlugin.lower() == 'all':
urlconfig.diyPlugin = LIST_PLUGINS #sessting里的设置插件
else:
urlconfig.diyPlugin = diyPlugin.strip().split(' ')
urlconfig.scanport = False
urlconfig.find_service = False
#是否开启端口和服务器信息扫描
if 'find_service' in urlconfig.diyPlugin:
urlconfig.find_service = True
input_scanport = input(
'[2.1] Need you scan all ports ?(Y/N) (default N)> ')
if input_scanport.lower() in ("y", "yes"):
urlconfig.scanport = True
def guideRegister(args):
if args.plugin and args.u:
return False
inputUrl = raw_input('[1] Input url > ')
if inputUrl is '':
raise ToolkitSystemException("You have to enter the url")
if inputUrl.startswith("@"):
urlconfig.mutiurl = True
fileName = inputUrl[1:]
try:
o = open(fileName, "r").readlines()
for url in o:
urlconfig.url.append(makeurl(url.strip()))
except IOError:
raise ToolkitSystemException("Filename:'%s' open faild" % fileName)
if len(o) == 0:
raise ToolkitSystemException("The target address is empty")
else:
urlconfig.url.append(makeurl(inputUrl))
printMessage('[Prompt] URL has been loaded:%d' % len(urlconfig.url))
printMessage("[Prompt] You can select these plugins (%s) or select all" %
(' '.join(LIST_PLUGINS)))
diyPlugin = raw_input("[2] Please select the required plugins > ")
if diyPlugin.lower() == 'all':
urlconfig.diyPlugin = LIST_PLUGINS
else:
urlconfig.diyPlugin = diyPlugin.strip().split(' ')
printMessage("[Prompt] You select the plugins:%s" %
(' '.join(urlconfig.diyPlugin)))
urlconfig.scanport = False
urlconfig.find_service = False
if 'find_service' in urlconfig.diyPlugin:
urlconfig.find_service = True
input_scanport = raw_input(
'[2.1] Need you scan all ports ?(Y/N) (default N)> ')
if input_scanport.lower() in ("y", "yes"):
urlconfig.scanport = True
urlconfig.threadNum = raw_input(
'[3] You need start number of thread (default 5) > ')
if urlconfig.threadNum == '':
urlconfig.threadNum = 5
urlconfig.threadNum = int(urlconfig.threadNum)
urlconfig.deepMax = raw_input(
'[4] Set the depth of the crawler (default 100 | 0 don\'t use crawler ) > '
)
if urlconfig.deepMax == '':
urlconfig.deepMax = 100
def configFileProxy(section, option, datatype):
"""
Parse configuration file and save settings into the configuration
advanced dictionary.
"""
global config
if config.has_option(section, option):
try:
#判断datatype类型 enums中定义了类型，get方法在ConfigParser
if datatype == OPTION_TYPE.BOOLEAN:
value = config.getboolean(section, option) if config.get(
section, option) else False
elif datatype == OPTION_TYPE.INTEGER:
value = config.getint(section, option) if config.get(
section, option) else 0
elif datatype == OPTION_TYPE.FLOAT:
value = config.getfloat(section, option) if config.get(
section, option) else 0.0
else:
value = config.get(section, option)
except ValueError, ex:
errMsg = "error occurred while processing the option "
errMsg += "'%s' in provided configuration file ('%s')" % (
option, getUnicode(ex))
raise ToolkitSystemException(errMsg)
if value:
Ajconfig[option] = value
else:
Ajconfig[option] = None
def checkFile(filename, raiseOnError=True):
"""
Checks for file existence and readability
"""
valid = True
try:
#判断文件是否存在
if filename is None or not os.path.isfile(filename):
valid = False
except UnicodeError:
valid = False
if valid:
try:
with open(filename, "rb"):
pass
except:
valid = False
if not valid and raiseOnError:
raise ToolkitSystemException("unable to read file '%s'" % filename)
return valid
def configFileParser(configFile):
"""
Parse configuration file and save settings into the configuration
advanced dictionary.
"""
global config
debugMsg = "parsing configuration file"
logger.debug(debugMsg)
checkFile(configFile)
configFP = openFile(configFile, "rb")
try:
config = UnicodeRawConfigParser()
config.readfp(configFP)
except Exception as ex:
errMsg = "you have provided an invalid and/or unreadable configuration file ('%s')" % ex
raise ToolkitSystemException(errMsg)
if not config.has_section("Config"):
errMsg = "missing a mandatory section 'Config' in the configuration file"
raise ToolkitSystemException(errMsg)
mandatory = False
for option in ("thread", "crawlerDeep", "TimeOut", "UserAgent", "Cookie",
"headers"):
if config.has_option("Config", option):
mandatory = True
break
if not mandatory:
errMsg = "missing a mandatory option in the configuration file "
errMsg += "(direct, url, logFile, bulkFile, googleDork, requestFile, sitemapUrl or wizard)"
raise ToolkitSystemException(errMsg)
for family, optionData in optDict.items():
for option, datatype in optionData.items():
datatype = unArrayizeValue(datatype)
configFileProxy(family, option, datatype)
def openFile(filename, mode='r', encoding="utf8", errors="replace", buffering=1): # "buffering=1" means line buffered (Reference: http://stackoverflow.com/a/3168436)
"""
Returns file handle of a given filename
"""
try:
return codecs.open(filename, mode, encoding, errors, buffering)
except IOError:
errMsg = "there has been a file opening error for filename '%s'. " % filename
errMsg += "Please check %s permissions on a file " % ("write" if \
mode and ('w' in mode or 'a' in mode or '+' in mode) else "read")
errMsg += "and that it's not locked by another process."
raise ToolkitSystemException(errMsg)
def configFileParser(configFile):
"""
Parse configuration file and save settings into the configuration
advanced dictionary.
"""
global config
debugMsg = "parsing configuration file"
logger.debug(debugMsg)
checkFile(configFile)
configFP = openFile(configFile, "rb")
try:
config = UnicodeRawConfigParser()
config.readfp(configFP) #ConfigParser
except Exception, ex:
errMsg = "you have provided an invalid and/or unreadable configuration file ('%s')" % ex
raise ToolkitSystemException(errMsg)
def configFileProxy(section, option, datatype):
"""
Parse configuration file and save settings into the configuration
advanced dictionary.
"""
global config
if config.has_option(section, option):
try:
if datatype == OPTION_TYPE.BOOLEAN:
value = config.getboolean(section, option) if config.get(
section, option) else False
elif datatype == OPTION_TYPE.INTEGER:
value = config.getint(section, option) if config.get(
section, option) else 0
elif datatype == OPTION_TYPE.FLOAT:
value = config.getfloat(section, option) if config.get(
section, option) else 0.0
else:
value = config.get(section, option)
except ValueError as ex:
errMsg = "error occurred while processing the option "
errMsg += "'%s' in provided configuration file ('%s')" % (
option, getUnicode(ex))
raise ToolkitSystemException(errMsg)
if value:
Ajconfig[option] = value
else:
Ajconfig[option] = None
else:
debugMsg = "missing requested option '%s' (section " % option
debugMsg += "'%s') into the configuration file, " % section
debugMsg += "ignoring. Skipping to next."
logger.debug(debugMsg)
debugMsg = "parsing configuration file"
logger.debug(debugMsg)
checkFile(configFile)
configFP = openFile(configFile, "rb")
try:
config = UnicodeRawConfigParser()
config.readfp(configFP) #ConfigParser
except Exception, ex:
errMsg = "you have provided an invalid and/or unreadable configuration file ('%s')" % ex
raise ToolkitSystemException(errMsg)
if not config.has_section("Config"):
errMsg = "missing a mandatory section 'Config' in the configuration file"
raise ToolkitSystemException(errMsg)
mandatory = False
for option in ("threadNum", "crawlerDeep", "TimeOut", "UserAgent",
"Cookie", "headers"):
if config.has_option("Config", option):
mandatory = True
break
if not mandatory:
errMsg = "missing a mandatory option in the configuration file "
errMsg += "(direct, url, logFile, bulkFile, googleDork, requestFile, sitemapUrl or wizard)"
raise ToolkitSystemException(errMsg)
for family, optionData in optDict.items(): #返回可遍历的(键, 值) 元组数组