def update_program(): success = False if not os.path.exists(os.path.join(paths.ROOT_PATH, ".git")): msg = "Have not a git repository. Please checkout the 'tentacle' repository " msg += "from GitHub (e.g. 'git clone --depth 1 %s tentacle')" % GIT_REPOSITORY logger.error(msg) else: msg = "Updating tentacle to the latest version from the gitHub repository." logger.sysinfo(msg) msg = "Tentacle will try to update itself using 'git' command." logger.sysinfo(msg) msg = "Update in progress..." logger.sysinfo(msg) try: process = subprocess.Popen( "git checkout . && git pull %s HEAD" % GIT_REPOSITORY, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=paths.ROOT_PATH ) # Reference: http://blog.stastnarodina.com/honza-en/spot/python-unicodeencodeerror/ poll_process(process, True) stdout, stderr = process.communicate() success = not process.returncode except (IOError, OSError) as ex: success = False stderr = get_safe_ex_string(ex) if success: logger.success("The latest revision '%s'" % (get_revision_number())) else: if isinstance(stderr, str): if "Not a git repository" in stderr: msg = "Not a valid git repository. Please checkout the 'orleven/tentacle' repository " msg += "from GitHub (e.g. 'git clone --depth 1 %s tentacle')" % GIT_REPOSITORY logger.error(msg) else: logger.error("Update could not be completed ('%s')" % re.sub(r"\W+", " ", stderr).strip()) else: logger.error("Update could not be completed. ") if not success: if sys.platform == 'win32': msg = "for Windows platform it's recommended " msg += "to use a GitHub for Windows client for updating " msg += "purposes (http://windows.github.com/) or just " msg += "download the latest snapshot from " msg += GIT_REPOSITORY else: msg = "For Linux platform it's required " msg += "to install a standard 'git' package (e.g.: 'sudo apt-get install git')" logger.sysinfo(msg)
def execute(self, statement, arguments=None): while True: try: if arguments: self.cursor.execute(statement, arguments) else: self.cursor.execute(statement) except sqlite3.OperationalError as ex: if not "locked" in get_safe_ex_string(ex): raise else: break if statement.lstrip().upper().startswith("SELECT"): return self.cursor.fetchall()
def curl(method, url, **kwargs): headers = kwargs.get('headers') if headers == None: headers = {} headers['Accept-Charset'] = 'GB2312,utf-8;q=0.7,*;q=0.7' headers[ 'Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' headers['Accept-Encoding'] = 'gzip, deflate, sdch, br' headers['Referer'] = url if 'User-Agent' not in headers.keys(): headers["User-Agent"] = random.choice(USER_AGENTS) # if 'X-Forwarded-For' not in headers.keys(): # headers['X-Forwarded-For'] = random_IP() kwargs.setdefault('headers', headers) timeout = int(conf['config']['basic']['timeout']) max_retries = int(conf['config']['basic']['max_retries']) if 'timeout' not in headers.keys(): kwargs.setdefault('timeout', timeout) kwargs.setdefault('verify', False) # if True: # try: # _proxies = { # 'http': 'http://127.0.0.1:8080/', # 'https': 'https://127.0.0.1:8080/' # } # kwargs.setdefault('proxies', _proxies) # except: # logger.error("Error http(s) proxy: %s or %s." % (conf['config']['proxy']['http_proxy'],conf['config']['proxy']['https_proxy'])) try: with requests.sessions.Session() as session: session.mount('http://', HTTPAdapter(max_retries=max_retries)) session.mount('https://', HTTPAdapter(max_retries=max_retries)) return session.request(method=method, url=url, **kwargs) except TooManyRedirects as e: kwargs.setdefault('allow_redirects', False) try: return request(method, url, **kwargs) except Exception as e: return None except (ConnectionError, TimeoutError, ReadTimeout, ChunkedEncodingError) as e: pass except Exception as e: logger.error("Curl error: %s for %s" % (get_safe_ex_string(e), url)) return None
def _load_module(self,module_name): module_spec = importlib.util.find_spec(module_name) if module_spec: try: module = importlib.import_module(module_name) if 'POC' not in dir(module): logger.error('Invalid POC script, Please check the script: %s' % module.__name__) else: return module except Exception as e: logger.error('Invalid POC script, Please check the script: %s' % module_name) logger.error(get_safe_ex_string(e)) else: logger.error('Can\'t load modual: %s.' % conf.module_path) return None
def flush(self): while True: self._cache_lock.acquire() if self._write_cache.qsize() > 0: data = self._write_cache.get() self._cache_lock.release() else: self._cache_lock.release() break retries = 0 while True: try: self._cache_lock.acquire() self.execute( "INSERT INTO storage (tid,flag,module_name, name, target_host,target_port,url,level,type,data,res,other) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (data['id'], data['flag'], data['module_name'], data['name'], data['target_host'], data['target_port'], data['url'], data['level'], data['type'], serialize_object( data['req']), serialize_object(data['res']), serialize_object(data['other']))) except sqlite3.DatabaseError as ex: if not os.path.exists(self.database): debugMsg = "session file '%s' does not exist" % self.database logger.debug(debugMsg) break if retries == 0: warnMsg = "there has been a problem while writing to " warnMsg += "the session file ('%s')" % get_safe_ex_string( ex) logger.warning(warnMsg) if retries >= 3: return else: retries += 1 time.sleep(1) self._cache_lock.release() else: self._cache_lock.release() break
async def do_scan(self, module: Script,target: Union[dict]) -> Iterable[dict]: records = [] func_name = self.pm.func_name parameter = self.pm.parameter flag = -1 try: poc = module.POC() poc.initialize(target['host'], target['port'], target['url'], parameter) func = getattr(poc, func_name) logger.debug( "Running %s:%s for %s:%s" % (module.__name__, func_name, poc.target_host, poc.target_port)) async with async_timeout.timeout(timeout=int(conf['basic']['timeout'])): await func() flag = poc.flag if poc.url != None: target['url'] = poc.url except AttributeError as e: if 'has no attribute \'POC\'' in get_safe_ex_string(e): logger.error('Invalid POC script, Please check the script: %s' % module.__name__, ) elif '\'POC\' object has no attribute' in get_safe_ex_string(e): logger.error('%s, Please check it in the script: %s' % (e, module.__name__)) elif 'Function is not exist.' in get_safe_ex_string(e): logger.error( 'Function is not exist, Please check \'%s\' in the script: %s' % ( func_name, module.__name__,)) else: self.errmsg = traceback.format_exc() logger.error(self.errmsg) logger.error("%s %s:%s for %s:%d" % (e, module.__name__, func_name, target['host'], target['port'])) self._error_task_count += 1 except KeyError as e: logger.error("Missing parameters: %s, please load parameters by -p. For example. -p %s=value" % ( e, str(e).replace('\'', ''))) self._error_task_count += 1 except (ConnectionResetError, ConnectionAbortedError, TimeoutError): flag = poc.flag except (CancelledError, ConnectionRefusedError, OSError): if target['status'] != None: target['status'] -= 1 else: target['status'] = -1 except Exception: self._error_task_count += 1 errmsg = traceback.format_exc() logger.error("Error for " + target['host'] + ":" + str(target['port']) + "\r\n"+ errmsg) finally: if conf.VERBOSE or flag >= 0: if poc.flag >= 0: self._find_task_count += 1 if module.__name__ == 'script.info.port_scan': target['status'] = 5 if len(poc.res) == 0 : poc.res = [{"info": None , "key": "port scan"}] for res in poc.res: target['service'] = res['info'] await self.vul_targets.put(target) else: target['status'] = 3 data = { "id": target['id'], "flag": poc.flag, 'module_name': module.__name__, 'func_name': func_name, "name": poc.name, 'target_host': poc.target_host, 'target_port': poc.target_port, 'url': poc.url, 'base_url': poc.base_url, 'level': poc.level, 'type': poc.type, "req": poc.req, "res": poc.res, "other": poc.other, } self.hashdb.insert(data) self.hashdb.flush() print_dic(data) records.append(data) logger.debug("Ending %s:%s for %s:%s" % (module.__name__, func_name, poc.target_host, poc.target_port)) return records
async def _request(self, method, url, **kwargs): headers = kwargs.get('headers') if headers == None: headers = {} if 'Accept' not in headers.keys(): headers[ "Accept"] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' if 'Accept-Charset' not in headers.keys(): headers["Accept-Charset"] = 'GB2312,utf-8;q=0.7,*;q=0.7' if 'Accept-Encoding' not in headers.keys(): # headers["Accept-Encoding"] = 'gzip, deflate, sdch, br' headers["Accept-Encoding"] = 'gzip, deflate, sdch' headers['Referer'] = url if 'User-Agent' not in headers.keys( ) or 'aiohttp' in headers["User-Agent"]: headers["User-Agent"] = random.choice(USER_AGENTS) # random_ip = random_IP() # if 'Client_IP' not in headers.keys(): # headers['Client_IP'] = random_ip # if 'X-Forwarded-For' not in headers.keys(): # headers['X-Forwarded-For'] = random_ip kwargs.setdefault('headers', headers) kwargs.setdefault('verify_ssl', False) if self._limit: await self._limit.wait_available() total = self._max_fail_retries if method.lower() == 'get' else 0 timeout = int(conf['basic']['timeout']) if 'timeout' not in kwargs.keys(): kwargs.setdefault('timeout', timeout) # if kwargs.get('proxy') is None: # try: # if True: # proxy = 'http://localhost:8080/' # kwargs.setdefault('proxy', proxy) # except KeyError as e: # logger.error("Load tentacle config error: %s, please check the config in tentacle.conf." % e) for count in range(total): resp = None try: resp = await super()._request(method, url, **kwargs) return resp except Exception as ex: pass logger.warning( 'request to {url} failed, retrying ({count} / {total})...') if resp: resp.close() await asyncio.sleep(self._retry_interval) try: return await super()._request(method, url, **kwargs) except TooManyRedirects: kwargs.setdefault('max_redirects', 3) try: return await self._request(method, url, **kwargs) except: return None except (ClientOSError, ClientResponseError, ClientConnectorError, ServerDisconnectedError): return None except Exception as e: if get_safe_ex_string(e).strip( ) != '' and 'InvalidServerVersion' not in get_safe_ex_string( e) and 'Unexpected SOCKS' not in get_safe_ex_string(e): # errmsg = traceback.format_exc() # logger.error(errmsg) logger.error("Curl error: %s for %s" % (get_safe_ex_string(e), url)) return None
def scan(self, id, module, obj): host, port, service, url = obj try: poc = module.POC() poc.initialize(host, port, url, self.parameter) func = getattr(poc, self.func_name) logger.debug("Running %s:%s for %s:%s" % (module.__name__, self.func_name, poc.target_host, poc.target_port)) func() logger.debug("Ending %s:%s for %s:%s" % (module.__name__, self.func_name, poc.target_host, poc.target_port)) if conf.VERBOSE or poc.flag >= 0: if poc.flag >= 1: self.change_found_count(1) if poc.flag == 2: if not conf[ 'noportscan'] or module.__name__ != 'script.info.port_scan': url = poc.url if poc.url != None else poc.base_url if poc.base_url != None else None service = poc.service_type[0] pool = self.put_queue_by_res( id, module, host, port, service, url) self.pools.append(pool) data = { "id": id, "flag": poc.flag, 'module_name': module.__name__, 'func_name': self.func_name, 'target_host': poc.target_host, 'target_port': poc.target_port, 'url': poc.url, 'base_url': poc.base_url, "req": poc.req, "res": poc.res, "other": poc.other, } self.hashdb.insert(data) self.hashdb.flush() print_dic(data) except AttributeError as e: if 'has no attribute \'POC\'' in get_safe_ex_string(e): logger.error( 'Invalid POC script, Please check the script: %s' % module.__name__, ) elif '\'POC\' object has no attribute' in get_safe_ex_string(e): logger.error('%s, Please check it in the script: %s' % (e, module.__name__)) elif 'Function is not exist.' in get_safe_ex_string(e): logger.error( 'Function is not exist, Please check \'%s\' in the script: %s' % ( self.func_name, module.__name__, )) else: logger.error("%s %s:%s for %s:%d" % (e, module.__name__, self.func_name, host, port)) self.change_error_count(1) except KeyError as e: logger.error( "Missing parameters: %s, please load parameters by -p. For example. -p %s=value" % (e, str(e).replace('\'', ''))) except Exception as e: self.errmsg = traceback.format_exc() self.is_continue = False logger.error(self.errmsg)