def run(self):
"""Initialize the context for the targeted service"""
logger.smartinfo('SmartStart processing to initialize context...')
# Detect if encrypted protocol (SSL/TLS) from original service name
# (from Nmap/Shodan)
processor = MatchstringsProcessor(self.service,
'service-name-original',
self.service.name_original, self.cu)
processor.detect_specific_options()
self.cu.update()
# Update context from banner
processor = MatchstringsProcessor(self.service, 'banner',
self.service.banner, self.cu)
processor.detect_products()
processor.detect_specific_options()
if not self.service.host.os:
processor.detect_os()
self.cu.update()
# Run start method corresponding to target service if available
list_methods = [method_name for method_name in dir(self) \
if callable(getattr(self, method_name))]
start_method_name = 'start_{}'.format(self.service.name)
if start_method_name in list_methods:
start_method = getattr(self, start_method_name)
start_method()
self.cu.update()
def __update_credentials(self, service):
for credential in self.credentials:
credential_str = '{username}/{password} {auth_type}'.format(
username=credential.username or '<empty>',
password=credential.password or '<empty>',
auth_type='(' + credential.type +
')' if credential.type else '')
match_cred = service.get_credential(credential.username,
credential.type)
if match_cred:
if match_cred.password is None:
logger.smartsuccess(
'Credentials found (username already known): ' +
credential_str)
match_cred.password = credential.password
elif match_cred.password != credential.password:
logger.smartsuccess('Credentials found (new password)' +
credential_str)
match_cred.password = credential.password
else:
logger.smartinfo('Credentials detected (no update): ' +
credential_str)
else:
logger.smartsuccess('New Credentials found: ' + credential_str)
service.credentials.append(credential)
def __update_credentials(self):
"""Update service's credentials (username+password) (in table "credentials")"""
for credential in self.credentials:
credential_str = '{username}/{password} {auth_type}'.format(
username=credential.username or '<empty>',
password=credential.password or '<empty>',
auth_type='(type='+credential.type+')' if credential.type else '')
match_cred = self.service.get_credential(
credential.username, credential.type)
if match_cred:
if match_cred.password is None:
logger.smartsuccess('Credentials found (username already ' \
'known): {}'.format(credential_str))
match_cred.password = credential.password
elif match_cred.password != credential.password:
logger.smartsuccess('Credentials found (new password): {}'.format(
credential_str))
match_cred.password = credential.password
else:
logger.smartinfo('Credentials detected (already in db): {}'.format(
credential_str))
else:
logger.smartsuccess('New Credentials found: {}'.format(credential_str))
self.service.credentials.append(credential)
def __update_usernames(self):
"""
Update service's usernames (in table "credentials").
This is called when only valid username has been found (but not the
associated password).
"""
for username in self.usernames:
username_str = '{username} {auth_type}'.format(
username=username.username or '<empty>',
auth_type='(type=' + username.type +
')' if username.type else '')
match_cred = self.service.get_credential(username.username,
username.type)
if match_cred:
if match_cred.password is None:
logger.smartinfo(
'Detected username (already known): {}'.format(
username_str))
else:
logger.smartinfo('Detected username (password already ' \
'known): {}'.format(username_str))
else:
logger.smartsuccess(
'New detected username: {}'.format(username_str))
self.service.credentials.append(username)
def run(self):
"""Run postcheck processing"""
logger.smartinfo('SmartPostcheck processing to update context...')
self.processor.detect_credentials()
self.processor.detect_specific_options()
self.processor.detect_products()
self.processor.detect_vulns()
self.cu.update()
def run(self):
"""Run postcheck processing"""
logger.smartinfo('SmartPostcheck processing to update context...')
self.cu = ContextUpdater(self.service, self.sqlsess)
self.__detect_credentials()
self.__detect_specific_options()
self.__detect_products()
self.__detect_vulns()
self.cu.update()
def __update_vulns(self):
"""Update service's vulnerabilities (table "vulns")"""
for vuln in self.vulns:
match_vuln = self.service.get_vuln(vuln.name)
if match_vuln:
logger.smartinfo('Detected vulnerability (already in db): {name}'.format(
name=vuln.name))
else:
logger.smartsuccess('New vulnerability detected: {name}'.format(
name=vuln.name))
self.service.vulns.append(vuln)
def call_start_method(self, service):
mod = self.__get_smartmodule(service.name)
if not mod:
return False
logger.smartinfo('Running initialization method...')
result = mod.start(service)
if result:
result.update_service(service)
self.sqlsess.commit()
return True
def run(self):
"""Run start method corresponding to target service if available"""
list_methods = [method_name for method_name in dir(self) \
if callable(getattr(self, method_name))]
start_method_name = 'start_{}'.format(self.service.name)
if start_method_name in list_methods:
logger.smartinfo('SmartStart processing to initialize context...')
start_method = getattr(self, start_method_name)
self.cu = ContextUpdater(self.service, self.sqlsess)
start_method()
self.cu.update()
def start_http(self):
"""Method run specifically for HTTP services"""
# Autodetect HTTPS
if self.service.url.lower().startswith('https://'):
logger.smartinfo('HTTPS protocol detected from URL')
self.cu.add_option('https', 'true')
# Check if HTTP service is protected by .htaccess authentication
if self.service.http_headers \
and '401 Unauthorized'.lower() in self.service.http_headers.lower():
logger.smartinfo('HTTP authentication (htaccess) detected ' \
'(401 Unauthorized)')
self.cu.add_option('htaccess', 'true')
# Update context with web technologies
if self.service.web_technos:
# Detect OS
if not self.service.host.os:
processor = MatchstringsProcessor(self.service, 'wappalyzer',
self.service.host.os,
self.cu)
processor.detect_os()
# Detect products
try:
technos = ast.literal_eval(self.service.web_technos)
except Exception as e:
logger.debug('Error when retrieving "web_technos" field ' \
'from db: {}'.format(e))
technos = list()
for t in technos:
for prodtype in products_match['http']:
p = products_match['http'][prodtype]
for prodname in p:
if 'wappalyzer' in p[prodname]:
pattern = p[prodname]['wappalyzer']
#m = re.search(pattern, t['name'], re.IGNORECASE|re.DOTALL)
if pattern.lower() == t['name'].lower():
version = t['version']
self.cu.add_product(prodtype, prodname,
version)
# Move to next product type if something found
break
def __detect_product_from_banner(self, prodtype):
"""
Detect product from Nmap banner.
:param str prodtype: Product type
"""
if self.service.banner:
p = products_match[self.service.name][prodtype]
for servername in p:
if 'nmap-banner' in p[servername]:
pattern = p[servername]['nmap-banner']
version_detection = '[VERSION]' in pattern
pattern = pattern.replace('[VERSION]', VERSION_REGEXP)
try:
m = re.search(pattern, self.service.banner,
re.IGNORECASE | re.DOTALL)
except Exception as e:
logger.warning('Error with matchstring [{pattern}], you should '\
'review it. Exception: {exception}'.format(
pattern=pattern, exception=e))
break
# If pattern matches banner, add detected product
if m:
# Add version if present
if version_detection:
try:
if m.group('version') is not None:
version = m.group('version')
else:
version = ''
except:
version = ''
else:
version = ''
logger.smartinfo('Product detected from banner: {type} = ' \
'{name} {version}'.format(
type=prodtype,
name=servername,
version=version))
# Add detected product to context
self.cu.add_product(prodtype, servername, version)
# Stop product detection from banner if something found
break
def __update_specific_options(self):
"""Update service's specific options (table "options")"""
for option in self.specific_options:
match_option = self.service.get_option(option.name)
if match_option:
if match_option.value == option.value:
logger.smartinfo('Detected option (already known): {name} = ' \
'{old}'.format(name=option.name, old=match_option.value))
else:
logger.smartsuccess('Change option: {name} = {old} -> {new}'.format(
name=option.name, old=match_option.value, new=option.value))
match_option.value = option.value
else:
logger.smartsuccess('New detected option: {name} = {new}'.format(
name=option.name, new=option.value))
self.service.options.append(option)
def __update_usernames(self, service):
for username in self.usernames:
username_str = '{username} {auth_type}'.format(
username=username.username or '<empty>',
auth_type='(' + username.type + ')' if username.type else '')
match_cred = service.get_credential(username.username,
username.type)
if match_cred:
if match_cred.password is None:
logger.smartinfo('Detected username (already knwon): ' +
username_str)
else:
logger.smartinfo(
'Detected username (password already known): ' +
username_str)
else:
logger.smartsuccess('New detected username: ' + username_str)
service.credentials.append(username)
def call_postcheck_method(self, method_name, service, cmd_output):
"""
:param service: Service object
"""
mod = self.__get_smartmodule(service.name)
if not mod or not mod.is_valid_postcheck_method(method_name):
return False
method = mod.get_postcheck_method(method_name)
logger.smartinfo('Running post-check method "{method}" ...'.format(
method=method_name))
result = method(cmd_output)
if result:
result.update_service(service)
self.sqlsess.commit()
return True
#loader = SmartModulesLoader()
#loader.callMethod('http', 'testMethod3', None)
def __update_specific_options(self, service):
for option in self.specific_options:
match_option = service.get_option(option.name)
if match_option:
if match_option.value == option.value:
logger.smartinfo(
'Detected option (no update): {name} = {old}'.format(
name=option.name, old=match_option.value))
else:
logger.smartsuccess(
'Change option: {name} = {old} -> {new}'.format(
name=option.name,
old=match_option.value,
new=option.value))
match_option.value = option.value
else:
logger.smartsuccess(
'New detected option: {name} = {new}'.format(
name=option.name, new=option.value))
service.options.append(option)
def start_http(self):
# Autodetect HTTPS
if self.service.url.lower().startswith('https://'):
logger.smartinfo('HTTPS protocol detected from URL')
self.cu.add_option('https', 'true')
# Check if HTTP service is protected by .htaccess authentication
if '401 Unauthorized'.lower() in self.service.http_headers.lower():
logger.smartinfo('HTTP authentication (htaccess) detected ' \
'(401 Unauthorized)')
self.cu.add_option('htaccess', 'true')
# Try to detect web server and/or appserver from Nmap banner
self.__detect_product_from_banner('web-server')
self.__detect_product_from_banner('web-appserver')
# Try to detect supported products from web technologies
if self.service.web_technos:
try:
technos = ast.literal_eval(self.service.web_technos)
except Exception as e:
logger.debug('Error when retrieving "web_technos" field ' \
'from db: {}'.format(e))
technos = list()
for t in technos:
for prodtype in products_match['http']:
p = products_match['http'][prodtype]
for prodname in p:
if 'wappalyzer' in p[prodname]:
pattern = p[prodname]['wappalyzer']
#m = re.search(pattern, t['name'], re.IGNORECASE|re.DOTALL)
if pattern.lower() == t['name'].lower():
version = t['version']
self.cu.add_product(prodtype, prodname,
version)
# Move to next product type if something found
break
def __update_products(self):
"""Update service's products (in table "products")"""
for product in self.products:
product_str = '{type}={name}'.format(
type=product.type,
name=product.name)
match_product = self.service.get_product(product.type)
# Same type already present in database
if match_product:
# Same product name detected
if match_product.name == product.name:
# Version detected
if product.version:
# Version freshly detected
if match_product.version == '':
logger.smartsuccess('Version detected for product ' \
'{product}: {version}'.format(
product=product_str,
version=product.version))
match_product.version = product.version
# Update version if new version is "more accurate" than the
# one already known
elif match_product.version != product.version:
if VersionUtils.is_version_more_precise(
old_version=match_product.version,
new_version=product.version):
logger.smartsuccess('Version for product ' \
'{product} updated: {oldvers} -> {newvers}'.format(
product=product_str,
oldvers=match_product.version,
newvers=product.version))
match_product.version = product.version
else:
logger.smartinfo('Version detected for product ' \
'{product}: {newvers}. Not updated in db ' \
'because less accurate than {oldvers}'.format(
product=product_str,
newvers=product.version,
oldvers=match_product.version))
# Version detected is superior (newer version) to the one in
# db, no update
# elif match_product.version < product.version:
# logger.smartsuccess('Version for product ' \
# '{product} detected: {newvers}. Not updated in db ' \
# 'because older version {oldvers} already detected'.format(
# product=product_str,
# newvers=product.version,
# oldvers=match_product.version))
# match_product.version = product.version
# Same version as already detected
else:
logger.smartinfo('Product detected: {product} ' \
'{version}. Not updated because already in db'.format(
product=product_str,
version=product.version))
# Version not detected
else:
logger.smartinfo('Product detected (already in db): ' \
'{product} (version unknown)'.format(product=product_str))
# Different product name detected
else:
oldprod = '{name}{vers}'.format(
name=match_product.name,
vers=' '+match_product.version if match_product.version else '')
newprod = '{name}{vers}'.format(
name=product.name,
vers=' '+product.version if product.version else '')
logger.smartsuccess('Change product {type}: {oldprod} -> ' \
'{newprod}'.format(
type=product.type,
oldprod=oldprod,
newprod=newprod))
match_product.name = product.name
match_product.version = product.version
# Type not present in database
else:
logger.smartsuccess('New product detected: {product} {version}'.format(
product=product_str,
version=product.version))
self.service.products.append(product)
def start(self, service):
# Mapping Nmap banner (lowercase) => context-specific option value
MAPPING_BANNER = {
'domino': 'lotusdomino',
}
# Mapping from Wappalyzer output (lowercase) => context-specific option value
MAPPING_WAPPALYZER = {
'apache-tomcat': 'tomcat',
'jboss-application-server': 'jboss',
'jboss-web': 'jboss',
'lotus-domino': 'lotusdomino',
'microsoft-asp.net': 'asp',
'adobe-coldfusion': 'coldfusion',
}
result = SmartModuleResult()
# Autodetect https
if service.url.lower().startswith('https://'):
logger.info('HTTPS protocol detected from URL')
result.add_option('https', 'true')
# Try to detect server from banner
if service.banner:
banner = service.banner.lower()
detected = None
for server in self.supported_list_options['server']:
if server in banner:
result.add_option('server', server)
detected = server
for server in MAPPING_BANNER.keys():
if server in banner:
result.add_option('server', server)
detected = server
if detected:
logger.info('Server detected from banner: {server}'.format(
server=detected))
# Autodetect web technos using Wappalyzer
try:
#print(WebPage(service.url).info())
technos = list(
map(lambda x: x.lower().replace(' ', '-'),
WebPage(service.url).info()['apps'].split(';')))
logger.smartinfo(
'Wappalyzer fingerprinting returns: {}'.format(technos))
for tech in technos:
if tech in MAPPING_WAPPALYZER.keys():
tech = MAPPING_WAPPALYZER[tech]
if tech in self.supported_list_options['language']:
result.add_option('language', tech)
elif tech in self.supported_list_options['cms']:
result.add_option('cms', tech)
elif tech in self.supported_list_options['server']:
result.add_option('server', tech)
except Exception as e:
logger.error('Wappalyzer error: {}'.format(e))
return result
