def run(self): """Initialize the context for the targeted service""" logger.smartinfo('SmartStart processing to initialize context...') # Detect if encrypted protocol (SSL/TLS) from original service name # (from Nmap/Shodan) processor = MatchstringsProcessor(self.service, 'service-name-original', self.service.name_original, self.cu) processor.detect_specific_options() self.cu.update() # Update context from banner processor = MatchstringsProcessor(self.service, 'banner', self.service.banner, self.cu) processor.detect_products() processor.detect_specific_options() if not self.service.host.os: processor.detect_os() self.cu.update() # Run start method corresponding to target service if available list_methods = [method_name for method_name in dir(self) \ if callable(getattr(self, method_name))] start_method_name = 'start_{}'.format(self.service.name) if start_method_name in list_methods: start_method = getattr(self, start_method_name) start_method() self.cu.update()
def __update_credentials(self, service): for credential in self.credentials: credential_str = '{username}/{password} {auth_type}'.format( username=credential.username or '<empty>', password=credential.password or '<empty>', auth_type='(' + credential.type + ')' if credential.type else '') match_cred = service.get_credential(credential.username, credential.type) if match_cred: if match_cred.password is None: logger.smartsuccess( 'Credentials found (username already known): ' + credential_str) match_cred.password = credential.password elif match_cred.password != credential.password: logger.smartsuccess('Credentials found (new password)' + credential_str) match_cred.password = credential.password else: logger.smartinfo('Credentials detected (no update): ' + credential_str) else: logger.smartsuccess('New Credentials found: ' + credential_str) service.credentials.append(credential)
def __update_credentials(self): """Update service's credentials (username+password) (in table "credentials")""" for credential in self.credentials: credential_str = '{username}/{password} {auth_type}'.format( username=credential.username or '<empty>', password=credential.password or '<empty>', auth_type='(type='+credential.type+')' if credential.type else '') match_cred = self.service.get_credential( credential.username, credential.type) if match_cred: if match_cred.password is None: logger.smartsuccess('Credentials found (username already ' \ 'known): {}'.format(credential_str)) match_cred.password = credential.password elif match_cred.password != credential.password: logger.smartsuccess('Credentials found (new password): {}'.format( credential_str)) match_cred.password = credential.password else: logger.smartinfo('Credentials detected (already in db): {}'.format( credential_str)) else: logger.smartsuccess('New Credentials found: {}'.format(credential_str)) self.service.credentials.append(credential)
def __update_usernames(self): """ Update service's usernames (in table "credentials"). This is called when only valid username has been found (but not the associated password). """ for username in self.usernames: username_str = '{username} {auth_type}'.format( username=username.username or '<empty>', auth_type='(type=' + username.type + ')' if username.type else '') match_cred = self.service.get_credential(username.username, username.type) if match_cred: if match_cred.password is None: logger.smartinfo( 'Detected username (already known): {}'.format( username_str)) else: logger.smartinfo('Detected username (password already ' \ 'known): {}'.format(username_str)) else: logger.smartsuccess( 'New detected username: {}'.format(username_str)) self.service.credentials.append(username)
def run(self): """Run postcheck processing""" logger.smartinfo('SmartPostcheck processing to update context...') self.processor.detect_credentials() self.processor.detect_specific_options() self.processor.detect_products() self.processor.detect_vulns() self.cu.update()
def run(self): """Run postcheck processing""" logger.smartinfo('SmartPostcheck processing to update context...') self.cu = ContextUpdater(self.service, self.sqlsess) self.__detect_credentials() self.__detect_specific_options() self.__detect_products() self.__detect_vulns() self.cu.update()
def __update_vulns(self): """Update service's vulnerabilities (table "vulns")""" for vuln in self.vulns: match_vuln = self.service.get_vuln(vuln.name) if match_vuln: logger.smartinfo('Detected vulnerability (already in db): {name}'.format( name=vuln.name)) else: logger.smartsuccess('New vulnerability detected: {name}'.format( name=vuln.name)) self.service.vulns.append(vuln)
def call_start_method(self, service): mod = self.__get_smartmodule(service.name) if not mod: return False logger.smartinfo('Running initialization method...') result = mod.start(service) if result: result.update_service(service) self.sqlsess.commit() return True
def run(self): """Run start method corresponding to target service if available""" list_methods = [method_name for method_name in dir(self) \ if callable(getattr(self, method_name))] start_method_name = 'start_{}'.format(self.service.name) if start_method_name in list_methods: logger.smartinfo('SmartStart processing to initialize context...') start_method = getattr(self, start_method_name) self.cu = ContextUpdater(self.service, self.sqlsess) start_method() self.cu.update()
def start_http(self): """Method run specifically for HTTP services""" # Autodetect HTTPS if self.service.url.lower().startswith('https://'): logger.smartinfo('HTTPS protocol detected from URL') self.cu.add_option('https', 'true') # Check if HTTP service is protected by .htaccess authentication if self.service.http_headers \ and '401 Unauthorized'.lower() in self.service.http_headers.lower(): logger.smartinfo('HTTP authentication (htaccess) detected ' \ '(401 Unauthorized)') self.cu.add_option('htaccess', 'true') # Update context with web technologies if self.service.web_technos: # Detect OS if not self.service.host.os: processor = MatchstringsProcessor(self.service, 'wappalyzer', self.service.host.os, self.cu) processor.detect_os() # Detect products try: technos = ast.literal_eval(self.service.web_technos) except Exception as e: logger.debug('Error when retrieving "web_technos" field ' \ 'from db: {}'.format(e)) technos = list() for t in technos: for prodtype in products_match['http']: p = products_match['http'][prodtype] for prodname in p: if 'wappalyzer' in p[prodname]: pattern = p[prodname]['wappalyzer'] #m = re.search(pattern, t['name'], re.IGNORECASE|re.DOTALL) if pattern.lower() == t['name'].lower(): version = t['version'] self.cu.add_product(prodtype, prodname, version) # Move to next product type if something found break
def __detect_product_from_banner(self, prodtype): """ Detect product from Nmap banner. :param str prodtype: Product type """ if self.service.banner: p = products_match[self.service.name][prodtype] for servername in p: if 'nmap-banner' in p[servername]: pattern = p[servername]['nmap-banner'] version_detection = '[VERSION]' in pattern pattern = pattern.replace('[VERSION]', VERSION_REGEXP) try: m = re.search(pattern, self.service.banner, re.IGNORECASE | re.DOTALL) except Exception as e: logger.warning('Error with matchstring [{pattern}], you should '\ 'review it. Exception: {exception}'.format( pattern=pattern, exception=e)) break # If pattern matches banner, add detected product if m: # Add version if present if version_detection: try: if m.group('version') is not None: version = m.group('version') else: version = '' except: version = '' else: version = '' logger.smartinfo('Product detected from banner: {type} = ' \ '{name} {version}'.format( type=prodtype, name=servername, version=version)) # Add detected product to context self.cu.add_product(prodtype, servername, version) # Stop product detection from banner if something found break
def __update_specific_options(self): """Update service's specific options (table "options")""" for option in self.specific_options: match_option = self.service.get_option(option.name) if match_option: if match_option.value == option.value: logger.smartinfo('Detected option (already known): {name} = ' \ '{old}'.format(name=option.name, old=match_option.value)) else: logger.smartsuccess('Change option: {name} = {old} -> {new}'.format( name=option.name, old=match_option.value, new=option.value)) match_option.value = option.value else: logger.smartsuccess('New detected option: {name} = {new}'.format( name=option.name, new=option.value)) self.service.options.append(option)
def __update_usernames(self, service): for username in self.usernames: username_str = '{username} {auth_type}'.format( username=username.username or '<empty>', auth_type='(' + username.type + ')' if username.type else '') match_cred = service.get_credential(username.username, username.type) if match_cred: if match_cred.password is None: logger.smartinfo('Detected username (already knwon): ' + username_str) else: logger.smartinfo( 'Detected username (password already known): ' + username_str) else: logger.smartsuccess('New detected username: ' + username_str) service.credentials.append(username)
def call_postcheck_method(self, method_name, service, cmd_output): """ :param service: Service object """ mod = self.__get_smartmodule(service.name) if not mod or not mod.is_valid_postcheck_method(method_name): return False method = mod.get_postcheck_method(method_name) logger.smartinfo('Running post-check method "{method}" ...'.format( method=method_name)) result = method(cmd_output) if result: result.update_service(service) self.sqlsess.commit() return True #loader = SmartModulesLoader() #loader.callMethod('http', 'testMethod3', None)
def __update_specific_options(self, service): for option in self.specific_options: match_option = service.get_option(option.name) if match_option: if match_option.value == option.value: logger.smartinfo( 'Detected option (no update): {name} = {old}'.format( name=option.name, old=match_option.value)) else: logger.smartsuccess( 'Change option: {name} = {old} -> {new}'.format( name=option.name, old=match_option.value, new=option.value)) match_option.value = option.value else: logger.smartsuccess( 'New detected option: {name} = {new}'.format( name=option.name, new=option.value)) service.options.append(option)
def start_http(self): # Autodetect HTTPS if self.service.url.lower().startswith('https://'): logger.smartinfo('HTTPS protocol detected from URL') self.cu.add_option('https', 'true') # Check if HTTP service is protected by .htaccess authentication if '401 Unauthorized'.lower() in self.service.http_headers.lower(): logger.smartinfo('HTTP authentication (htaccess) detected ' \ '(401 Unauthorized)') self.cu.add_option('htaccess', 'true') # Try to detect web server and/or appserver from Nmap banner self.__detect_product_from_banner('web-server') self.__detect_product_from_banner('web-appserver') # Try to detect supported products from web technologies if self.service.web_technos: try: technos = ast.literal_eval(self.service.web_technos) except Exception as e: logger.debug('Error when retrieving "web_technos" field ' \ 'from db: {}'.format(e)) technos = list() for t in technos: for prodtype in products_match['http']: p = products_match['http'][prodtype] for prodname in p: if 'wappalyzer' in p[prodname]: pattern = p[prodname]['wappalyzer'] #m = re.search(pattern, t['name'], re.IGNORECASE|re.DOTALL) if pattern.lower() == t['name'].lower(): version = t['version'] self.cu.add_product(prodtype, prodname, version) # Move to next product type if something found break
def __update_products(self): """Update service's products (in table "products")""" for product in self.products: product_str = '{type}={name}'.format( type=product.type, name=product.name) match_product = self.service.get_product(product.type) # Same type already present in database if match_product: # Same product name detected if match_product.name == product.name: # Version detected if product.version: # Version freshly detected if match_product.version == '': logger.smartsuccess('Version detected for product ' \ '{product}: {version}'.format( product=product_str, version=product.version)) match_product.version = product.version # Update version if new version is "more accurate" than the # one already known elif match_product.version != product.version: if VersionUtils.is_version_more_precise( old_version=match_product.version, new_version=product.version): logger.smartsuccess('Version for product ' \ '{product} updated: {oldvers} -> {newvers}'.format( product=product_str, oldvers=match_product.version, newvers=product.version)) match_product.version = product.version else: logger.smartinfo('Version detected for product ' \ '{product}: {newvers}. Not updated in db ' \ 'because less accurate than {oldvers}'.format( product=product_str, newvers=product.version, oldvers=match_product.version)) # Version detected is superior (newer version) to the one in # db, no update # elif match_product.version < product.version: # logger.smartsuccess('Version for product ' \ # '{product} detected: {newvers}. Not updated in db ' \ # 'because older version {oldvers} already detected'.format( # product=product_str, # newvers=product.version, # oldvers=match_product.version)) # match_product.version = product.version # Same version as already detected else: logger.smartinfo('Product detected: {product} ' \ '{version}. Not updated because already in db'.format( product=product_str, version=product.version)) # Version not detected else: logger.smartinfo('Product detected (already in db): ' \ '{product} (version unknown)'.format(product=product_str)) # Different product name detected else: oldprod = '{name}{vers}'.format( name=match_product.name, vers=' '+match_product.version if match_product.version else '') newprod = '{name}{vers}'.format( name=product.name, vers=' '+product.version if product.version else '') logger.smartsuccess('Change product {type}: {oldprod} -> ' \ '{newprod}'.format( type=product.type, oldprod=oldprod, newprod=newprod)) match_product.name = product.name match_product.version = product.version # Type not present in database else: logger.smartsuccess('New product detected: {product} {version}'.format( product=product_str, version=product.version)) self.service.products.append(product)
def start(self, service): # Mapping Nmap banner (lowercase) => context-specific option value MAPPING_BANNER = { 'domino': 'lotusdomino', } # Mapping from Wappalyzer output (lowercase) => context-specific option value MAPPING_WAPPALYZER = { 'apache-tomcat': 'tomcat', 'jboss-application-server': 'jboss', 'jboss-web': 'jboss', 'lotus-domino': 'lotusdomino', 'microsoft-asp.net': 'asp', 'adobe-coldfusion': 'coldfusion', } result = SmartModuleResult() # Autodetect https if service.url.lower().startswith('https://'): logger.info('HTTPS protocol detected from URL') result.add_option('https', 'true') # Try to detect server from banner if service.banner: banner = service.banner.lower() detected = None for server in self.supported_list_options['server']: if server in banner: result.add_option('server', server) detected = server for server in MAPPING_BANNER.keys(): if server in banner: result.add_option('server', server) detected = server if detected: logger.info('Server detected from banner: {server}'.format( server=detected)) # Autodetect web technos using Wappalyzer try: #print(WebPage(service.url).info()) technos = list( map(lambda x: x.lower().replace(' ', '-'), WebPage(service.url).info()['apps'].split(';'))) logger.smartinfo( 'Wappalyzer fingerprinting returns: {}'.format(technos)) for tech in technos: if tech in MAPPING_WAPPALYZER.keys(): tech = MAPPING_WAPPALYZER[tech] if tech in self.supported_list_options['language']: result.add_option('language', tech) elif tech in self.supported_list_options['cms']: result.add_option('cms', tech) elif tech in self.supported_list_options['server']: result.add_option('server', tech) except Exception as e: logger.error('Wappalyzer error: {}'.format(e)) return result