def update_program():
success = False
if not os.path.exists(os.path.join(paths.ROOT_PATH, ".git")):
msg = "Have not a git repository. Please checkout the 'tentacle' repository "
msg += "from GitHub (e.g. 'git clone --depth 1 %s tentacle')" % GIT_REPOSITORY
logger.error(msg)
else:
msg = "Updating tentacle to the latest version from the gitHub repository."
logger.sysinfo(msg)
msg = "Tentacle will try to update itself using 'git' command."
logger.sysinfo(msg)
msg = "Update in progress..."
logger.sysinfo(msg)
try:
process = subprocess.Popen(
"git checkout . && git pull %s HEAD" % GIT_REPOSITORY,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=paths.ROOT_PATH
) # Reference: http://blog.stastnarodina.com/honza-en/spot/python-unicodeencodeerror/
poll_process(process, True)
stdout, stderr = process.communicate()
success = not process.returncode
except (IOError, OSError) as ex:
success = False
stderr = get_safe_ex_string(ex)
if success:
logger.success("The latest revision '%s'" % (get_revision_number()))
else:
if isinstance(stderr, str):
if "Not a git repository" in stderr:
msg = "Not a valid git repository. Please checkout the 'orleven/tentacle' repository "
msg += "from GitHub (e.g. 'git clone --depth 1 %s tentacle')" % GIT_REPOSITORY
logger.error(msg)
else:
logger.error("Update could not be completed ('%s')" %
re.sub(r"\W+", " ", stderr).strip())
else:
logger.error("Update could not be completed. ")
if not success:
if sys.platform == 'win32':
msg = "for Windows platform it's recommended "
msg += "to use a GitHub for Windows client for updating "
msg += "purposes (http://windows.github.com/) or just "
msg += "download the latest snapshot from "
msg += GIT_REPOSITORY
else:
msg = "For Linux platform it's required "
msg += "to install a standard 'git' package (e.g.: 'sudo apt-get install git')"
logger.sysinfo(msg)
def execute(self, statement, arguments=None):
while True:
try:
if arguments:
self.cursor.execute(statement, arguments)
else:
self.cursor.execute(statement)
except sqlite3.OperationalError as ex:
if not "locked" in get_safe_ex_string(ex):
raise
else:
break
if statement.lstrip().upper().startswith("SELECT"):
return self.cursor.fetchall()
def curl(method, url, **kwargs):
headers = kwargs.get('headers')
if headers == None:
headers = {}
headers['Accept-Charset'] = 'GB2312,utf-8;q=0.7,*;q=0.7'
headers[
'Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
headers['Accept-Encoding'] = 'gzip, deflate, sdch, br'
headers['Referer'] = url
if 'User-Agent' not in headers.keys():
headers["User-Agent"] = random.choice(USER_AGENTS)
# if 'X-Forwarded-For' not in headers.keys():
# headers['X-Forwarded-For'] = random_IP()
kwargs.setdefault('headers', headers)
timeout = int(conf['config']['basic']['timeout'])
max_retries = int(conf['config']['basic']['max_retries'])
if 'timeout' not in headers.keys():
kwargs.setdefault('timeout', timeout)
kwargs.setdefault('verify', False)
# if True:
# try:
# _proxies = {
# 'http': 'http://127.0.0.1:8080/',
# 'https': 'https://127.0.0.1:8080/'
# }
# kwargs.setdefault('proxies', _proxies)
# except:
# logger.error("Error http(s) proxy: %s or %s." % (conf['config']['proxy']['http_proxy'],conf['config']['proxy']['https_proxy']))
try:
with requests.sessions.Session() as session:
session.mount('http://', HTTPAdapter(max_retries=max_retries))
session.mount('https://', HTTPAdapter(max_retries=max_retries))
return session.request(method=method, url=url, **kwargs)
except TooManyRedirects as e:
kwargs.setdefault('allow_redirects', False)
try:
return request(method, url, **kwargs)
except Exception as e:
return None
except (ConnectionError, TimeoutError, ReadTimeout,
ChunkedEncodingError) as e:
pass
except Exception as e:
logger.error("Curl error: %s for %s" % (get_safe_ex_string(e), url))
return None
def _load_module(self,module_name):
module_spec = importlib.util.find_spec(module_name)
if module_spec:
try:
module = importlib.import_module(module_name)
if 'POC' not in dir(module):
logger.error('Invalid POC script, Please check the script: %s' % module.__name__)
else:
return module
except Exception as e:
logger.error('Invalid POC script, Please check the script: %s' % module_name)
logger.error(get_safe_ex_string(e))
else:
logger.error('Can\'t load modual: %s.' % conf.module_path)
return None
def flush(self):
while True:
self._cache_lock.acquire()
if self._write_cache.qsize() > 0:
data = self._write_cache.get()
self._cache_lock.release()
else:
self._cache_lock.release()
break
retries = 0
while True:
try:
self._cache_lock.acquire()
self.execute(
"INSERT INTO storage (tid,flag,module_name, name, target_host,target_port,url,level,type,data,res,other) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
(data['id'], data['flag'], data['module_name'],
data['name'], data['target_host'],
data['target_port'], data['url'], data['level'],
data['type'], serialize_object(
data['req']), serialize_object(data['res']),
serialize_object(data['other'])))
except sqlite3.DatabaseError as ex:
if not os.path.exists(self.database):
debugMsg = "session file '%s' does not exist" % self.database
logger.debug(debugMsg)
break
if retries == 0:
warnMsg = "there has been a problem while writing to "
warnMsg += "the session file ('%s')" % get_safe_ex_string(
ex)
logger.warning(warnMsg)
if retries >= 3:
return
else:
retries += 1
time.sleep(1)
self._cache_lock.release()
else:
self._cache_lock.release()
break
async def do_scan(self, module: Script,target: Union[dict]) -> Iterable[dict]:
records = []
func_name = self.pm.func_name
parameter = self.pm.parameter
flag = -1
try:
poc = module.POC()
poc.initialize(target['host'], target['port'], target['url'], parameter)
func = getattr(poc, func_name)
logger.debug(
"Running %s:%s for %s:%s" % (module.__name__, func_name, poc.target_host, poc.target_port))
async with async_timeout.timeout(timeout=int(conf['basic']['timeout'])):
await func()
flag = poc.flag
if poc.url != None:
target['url'] = poc.url
except AttributeError as e:
if 'has no attribute \'POC\'' in get_safe_ex_string(e):
logger.error('Invalid POC script, Please check the script: %s' % module.__name__, )
elif '\'POC\' object has no attribute' in get_safe_ex_string(e):
logger.error('%s, Please check it in the script: %s' % (e, module.__name__))
elif 'Function is not exist.' in get_safe_ex_string(e):
logger.error(
'Function is not exist, Please check \'%s\' in the script: %s' % (
func_name, module.__name__,))
else:
self.errmsg = traceback.format_exc()
logger.error(self.errmsg)
logger.error("%s %s:%s for %s:%d" % (e, module.__name__, func_name, target['host'], target['port']))
self._error_task_count += 1
except KeyError as e:
logger.error("Missing parameters: %s, please load parameters by -p. For example. -p %s=value" % (
e, str(e).replace('\'', '')))
self._error_task_count += 1
except (ConnectionResetError, ConnectionAbortedError, TimeoutError):
flag = poc.flag
except (CancelledError, ConnectionRefusedError, OSError):
if target['status'] != None:
target['status'] -= 1
else:
target['status'] = -1
except Exception:
self._error_task_count += 1
errmsg = traceback.format_exc()
logger.error("Error for " + target['host'] + ":" + str(target['port']) + "\r\n"+ errmsg)
finally:
if conf.VERBOSE or flag >= 0:
if poc.flag >= 0:
self._find_task_count += 1
if module.__name__ == 'script.info.port_scan':
target['status'] = 5
if len(poc.res) == 0 :
poc.res = [{"info": None , "key": "port scan"}]
for res in poc.res:
target['service'] = res['info']
await self.vul_targets.put(target)
else:
target['status'] = 3
data = {
"id": target['id'],
"flag": poc.flag,
'module_name': module.__name__,
'func_name': func_name,
"name": poc.name,
'target_host': poc.target_host,
'target_port': poc.target_port,
'url': poc.url,
'base_url': poc.base_url,
'level': poc.level,
'type': poc.type,
"req": poc.req,
"res": poc.res,
"other": poc.other,
}
self.hashdb.insert(data)
self.hashdb.flush()
print_dic(data)
records.append(data)
logger.debug("Ending %s:%s for %s:%s" % (module.__name__, func_name, poc.target_host, poc.target_port))
return records
async def _request(self, method, url, **kwargs):
headers = kwargs.get('headers')
if headers == None:
headers = {}
if 'Accept' not in headers.keys():
headers[
"Accept"] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
if 'Accept-Charset' not in headers.keys():
headers["Accept-Charset"] = 'GB2312,utf-8;q=0.7,*;q=0.7'
if 'Accept-Encoding' not in headers.keys():
# headers["Accept-Encoding"] = 'gzip, deflate, sdch, br'
headers["Accept-Encoding"] = 'gzip, deflate, sdch'
headers['Referer'] = url
if 'User-Agent' not in headers.keys(
) or 'aiohttp' in headers["User-Agent"]:
headers["User-Agent"] = random.choice(USER_AGENTS)
# random_ip = random_IP()
# if 'Client_IP' not in headers.keys():
# headers['Client_IP'] = random_ip
# if 'X-Forwarded-For' not in headers.keys():
# headers['X-Forwarded-For'] = random_ip
kwargs.setdefault('headers', headers)
kwargs.setdefault('verify_ssl', False)
if self._limit:
await self._limit.wait_available()
total = self._max_fail_retries if method.lower() == 'get' else 0
timeout = int(conf['basic']['timeout'])
if 'timeout' not in kwargs.keys():
kwargs.setdefault('timeout', timeout)
# if kwargs.get('proxy') is None:
# try:
# if True:
# proxy = 'http://localhost:8080/'
# kwargs.setdefault('proxy', proxy)
# except KeyError as e:
# logger.error("Load tentacle config error: %s, please check the config in tentacle.conf." % e)
for count in range(total):
resp = None
try:
resp = await super()._request(method, url, **kwargs)
return resp
except Exception as ex:
pass
logger.warning(
'request to {url} failed, retrying ({count} / {total})...')
if resp:
resp.close()
await asyncio.sleep(self._retry_interval)
try:
return await super()._request(method, url, **kwargs)
except TooManyRedirects:
kwargs.setdefault('max_redirects', 3)
try:
return await self._request(method, url, **kwargs)
except:
return None
except (ClientOSError, ClientResponseError, ClientConnectorError,
ServerDisconnectedError):
return None
except Exception as e:
if get_safe_ex_string(e).strip(
) != '' and 'InvalidServerVersion' not in get_safe_ex_string(
e) and 'Unexpected SOCKS' not in get_safe_ex_string(e):
# errmsg = traceback.format_exc()
# logger.error(errmsg)
logger.error("Curl error: %s for %s" %
(get_safe_ex_string(e), url))
return None
def scan(self, id, module, obj):
host, port, service, url = obj
try:
poc = module.POC()
poc.initialize(host, port, url, self.parameter)
func = getattr(poc, self.func_name)
logger.debug("Running %s:%s for %s:%s" %
(module.__name__, self.func_name, poc.target_host,
poc.target_port))
func()
logger.debug("Ending %s:%s for %s:%s" %
(module.__name__, self.func_name, poc.target_host,
poc.target_port))
if conf.VERBOSE or poc.flag >= 0:
if poc.flag >= 1:
self.change_found_count(1)
if poc.flag == 2:
if not conf[
'noportscan'] or module.__name__ != 'script.info.port_scan':
url = poc.url if poc.url != None else poc.base_url if poc.base_url != None else None
service = poc.service_type[0]
pool = self.put_queue_by_res(
id, module, host, port, service, url)
self.pools.append(pool)
data = {
"id": id,
"flag": poc.flag,
'module_name': module.__name__,
'func_name': self.func_name,
'target_host': poc.target_host,
'target_port': poc.target_port,
'url': poc.url,
'base_url': poc.base_url,
"req": poc.req,
"res": poc.res,
"other": poc.other,
}
self.hashdb.insert(data)
self.hashdb.flush()
print_dic(data)
except AttributeError as e:
if 'has no attribute \'POC\'' in get_safe_ex_string(e):
logger.error(
'Invalid POC script, Please check the script: %s' %
module.__name__, )
elif '\'POC\' object has no attribute' in get_safe_ex_string(e):
logger.error('%s, Please check it in the script: %s' %
(e, module.__name__))
elif 'Function is not exist.' in get_safe_ex_string(e):
logger.error(
'Function is not exist, Please check \'%s\' in the script: %s'
% (
self.func_name,
module.__name__,
))
else:
logger.error("%s %s:%s for %s:%d" %
(e, module.__name__, self.func_name, host, port))
self.change_error_count(1)
except KeyError as e:
logger.error(
"Missing parameters: %s, please load parameters by -p. For example. -p %s=value"
% (e, str(e).replace('\'', '')))
except Exception as e:
self.errmsg = traceback.format_exc()
self.is_continue = False
logger.error(self.errmsg)