def main():
my_envs = os.environ
# If the script is run outside the dispatcher
# the environment variables
# are checked.
tool = os.environ.get("EXECUTOR_CONFIG_TOOL", None)
if "EXECUTOR_CONFIG_REPORT_NAME" in my_envs:
report_name = os.environ.get("EXECUTOR_CONFIG_REPORT_NAME")
else:
print("Argument REPORT_NAME no set", file=sys.stderr)
sys.exit()
if "REPORTS_PATH" in my_envs:
report_dir = os.environ.get("REPORTS_PATH")
else:
print("Environment variable REPORT_DIR no set", file=sys.stderr)
sys.exit()
filepath = Path(report_dir) / report_name
manager = PluginsManager()
if tool is not None:
plugin = manager.get_plugin(tool)
else:
plugin = ReportAnalyzer(manager).get_plugin(filepath)
print(f"Detector found it as an {plugin} report", file=sys.stderr)
print(f"Parsing {filepath} report", file=sys.stderr)
plugin.processReport(str(filepath))
print(plugin.get_json())
def main():
# If the script is run outside the dispatcher the environment variables
# are checked.
# ['EXECUTOR_CONFIG_API_KEY', 'EXECUTOR_CONFIG_TARGET_URL']
try:
target = os.environ["EXECUTOR_CONFIG_TARGET_URL"]
api_key = os.environ["EXECUTOR_CONFIG_API_KEY"]
except KeyError:
print("environment variable not found", file=sys.stderr)
sys.exit()
# zap is required to be started
zap_run = subprocess.check_output("pgrep -f zap", shell=True)
if len(zap_run.decode("utf-8").split("\n")) > 3:
# the apikey from ZAP->Tools->Options-API
zap = ZAPv2(apikey=api_key)
# it passes the url to scan and starts
scanID = zap.spider.scan(target)
# Wait for the scan to finish
while int(zap.spider.status(scanID)) < 100:
time.sleep(1)
# If finish the scan and the xml is generated
zap_result = zap.core.xmlreport()
plugin = PluginsManager().get_plugin("zap")
plugin.parseOutputString(zap_result)
print(plugin.get_json())
else:
print("ZAP not running", file=sys.stderr)
sys.exit()
def list_plugins(custom_plugins_folder):
plugins_manager = PluginsManager(custom_plugins_folder)
click.echo(click.style(f"Faraday Plugins v{__version__}", fg="cyan"))
click.echo(click.style("Available Plugins :", fg="cyan"))
loaded_plugins = 0
plugins_data = []
for plugin_id, plugin in plugins_manager.get_plugins():
console_enabled = plugin._command_regex is not None
console_enabled_color = "green" if console_enabled else "red"
console_enabled_text = click.style(
f"{'Yes' if console_enabled else 'No'}", fg=console_enabled_color)
report_enabled = isinstance(plugin, PluginByExtension)
report_enabled_color = "green" if report_enabled else "red"
report_enabled_text = click.style(
f"{'Yes' if report_enabled else 'No'}", fg=report_enabled_color)
plugins_data.append({
"Name": plugin.name,
"ID": plugin.id,
"Command": console_enabled_text,
"Report": report_enabled_text
})
click.echo(tabulate(
plugins_data,
headers="keys",
tablefmt="simple",
))
click.echo(click.style(f"Loaded Plugins: {len(plugins_data)}", fg="cyan"))
def process_report(report_file, plugin_id, custom_plugins_folder, summary,
output_file, ignore_info):
if not os.path.isfile(report_file):
click.echo(click.style(f"File {report_file} Don't Exists", fg="red"),
err=True)
else:
plugins_manager = PluginsManager(custom_plugins_folder,
ignore_info=ignore_info)
analyzer = ReportAnalyzer(plugins_manager)
if plugin_id:
plugin = plugins_manager.get_plugin(plugin_id)
if not plugin:
click.echo(click.style(f"Invalid Plugin: {plugin_id}",
fg="red"),
err=True)
return
else:
plugin = analyzer.get_plugin(report_file)
if not plugin:
click.echo(click.style(
f"Failed to detect report: {report_file}", fg="red"),
err=True)
return
plugin.processReport(Path(report_file), getpass.getuser())
if summary:
click.echo(json.dumps(plugin.get_summary(), indent=4))
else:
if output_file:
with open(output_file, "w") as f:
json.dump(plugin.get_data(), f)
else:
click.echo(json.dumps(plugin.get_data(), indent=4))
def __init__(self, upload_reports_queue, *args, **kwargs):
super().__init__(*args, **kwargs)
self.upload_reports_queue = upload_reports_queue
self.plugins_manager = PluginsManager(
config.faraday_server.custom_plugins_folder,
ignore_info=config.faraday_server.ignore_info_severity)
self.__event = threading.Event()
def process_command(command, plugin_id, custom_plugins_folder, dont_run,
summary, output_file, show_output, ignore_info):
plugins_manager = PluginsManager(custom_plugins_folder,
ignore_info=ignore_info)
analyzer = CommandAnalyzer(plugins_manager)
if plugin_id:
plugin = plugins_manager.get_plugin(plugin_id)
if not plugin:
click.echo(click.style(f"Invalid Plugin: {plugin_id}", fg="red"),
err=True)
return
else:
plugin = analyzer.get_plugin(command)
if not plugin:
click.echo(click.style(f"Failed to detect command: {command}",
fg="red"),
err=True)
return
current_path = os.path.abspath(os.getcwd())
modified_command = plugin.processCommandString(getpass.getuser(),
current_path, command)
if modified_command:
command = modified_command
if dont_run:
color_message = click.style("Command: ", fg="green")
click.echo(f"{color_message} {command}")
else:
p = subprocess.Popen(shlex.split(command),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
output = io.StringIO()
while True:
retcode = p.poll()
line = p.stdout.readline().decode('utf-8')
if show_output:
sys.stdout.write(line)
output.write(line)
if retcode is not None:
extra_lines = map(lambda x: x.decode('utf-8'),
p.stdout.readlines())
if show_output:
sys.stdout.writelines(line)
output.writelines(extra_lines)
break
output_value = output.getvalue()
if retcode == 0:
plugin.processOutput(output_value)
if summary:
click.echo(json.dumps(plugin.get_summary(), indent=4))
else:
if output_file:
with open(output_file, "w") as f:
json.dump(plugin.get_data(), f)
else:
click.echo(json.dumps(plugin.get_data(), indent=4))
else:
click.echo(click.style("Command execution error!!", fg="red"),
err=True)
def main():
# If the script is run outside the dispatcher the environment variables
# are checked.
# ['EXECUTOR_CONFIG_TOKEN', 'EXECUTOR_CONFIG_URL', 'EXECUTOR_CONFIG_PROJECT']
try:
sonar_qube_url = os.environ["EXECUTOR_CONFIG_URL"]
token = os.environ["EXECUTOR_CONFIG_TOKEN"]
component_key = os.environ['EXECUTOR_CONFIG_COMPONENT_KEY']
except KeyError:
print("Environment variable not found", file=sys.stderr)
sys.exit()
session = requests.Session()
# ATTENTION: SonarQube API requires an empty password when auth method is via token
session.auth = (token, '')
# Issues api config
page = 0
has_more_vulns = True
vulnerabilities = []
while has_more_vulns:
page += 1
params = {
'componentKeys': component_key,
'types': TYPE_VULNS,
'p': page,
'ps': PAGE_SIZE
}
response = session.get(
url=f'{sonar_qube_url}/api/issues/search',
params=params,
)
if not response.ok:
print(
f"There was an error finding issues. Component Key {component_key}; Status Code {response.status_code} - {response.content}",
file=sys.stderr)
sys.exit()
response_json = response.json()
issues = response_json.get('issues')
vulnerabilities.extend(issues)
total_items = response_json.get('paging').get('total')
has_more_vulns = page * PAGE_SIZE < total_items
plugin = PluginsManager().get_plugin("sonarqubeapi")
vulns_json = json.dumps({'issues': vulnerabilities})
plugin.parseOutputString(vulns_json)
print(plugin.get_json())
def __init__(self, plugin_repo_path, pending_actions=None):
self._controllers = {}
self._plugin_modules = {}
self._plugin_instances = {}
self._plugin_settings = {}
self.pending_actions = pending_actions
self._plugins_manager = PluginsManager(CONF.getCustomPluginsPath())
self.commands_analyzer = CommandAnalyzer(self._plugins_manager)
self.report_analyzer = ReportAnalyzer(self._plugins_manager)
self._loadSettings()
def process_report(workspace_name: str, command_id: int, file_path: Path,
plugin_id: Optional[int], user_id: Optional[int]):
from faraday.server.web import get_app # pylint:disable=import-outside-toplevel
with get_app().app_context():
if plugin_id is not None:
plugins_manager = PluginsManager(
ReportsSettings.settings.custom_plugins_folder,
ignore_info=ReportsSettings.settings.ignore_info_severity)
logger.info(
f"Reports Manager: [Custom plugins folder: "
f"[{ReportsSettings.settings.custom_plugins_folder}]"
f"[Ignore info severity: {ReportsSettings.settings.ignore_info_severity}]"
)
plugin = plugins_manager.get_plugin(plugin_id)
if plugin:
try:
logger.info(
f"Processing report [{file_path}] with plugin ["
f"{plugin.id}]")
plugin.processReport(str(file_path))
vulns_data = plugin.get_data()
del vulns_data['command']['duration']
except Exception as e:
logger.error("Processing Error: %s", e)
logger.exception(e)
return
else:
logger.error(f"No plugin detected for report [{file_path}]")
return
else:
try:
with file_path.open("r") as f:
vulns_data = json.load(f)
except Exception as e:
logger.error("Loading data from json file: %s [%s]", file_path,
e)
logger.exception(e)
return
if plugin_id is None:
logger.debug("Removing file: %s", file_path)
os.remove(file_path)
else:
if faraday_server.delete_report_after_process:
os.remove(file_path)
set_end_date = True
try:
send_report_data(workspace_name, command_id, vulns_data, user_id,
set_end_date)
logger.info("Report processing finished")
except Exception as e:
logger.exception(e)
logger.error("Save Error: %s", e)
def main():
NESSUS_SCAN_NAME = os.getenv("EXECUTOR_CONFIG_NESSUS_SCAN_NAME",
get_report_name())
NESSUS_URL = os.getenv("EXECUTOR_CONFIG_NESSUS_URL") # https://nessus:port
NESSUS_USERNAME = os.getenv("NESSUS_USERNAME")
NESSUS_PASSWORD = os.getenv("NESSUS_PASSWORD")
NESSUS_SCAN_TARGET = os.getenv(
# ip, domain, range
"EXECUTOR_CONFIG_NESSUS_SCAN_TARGET")
NESSUS_SCAN_TEMPLATE = os.getenv(
# name field
"EXECUTOR_CONFIG_NESSUS_SCAN_TEMPLATE",
"basic",
)
if not NESSUS_URL:
NESSUS_URL = os.getenv("NESSUS_URL")
if not NESSUS_URL:
print("URL not provided", file=sys.stderr)
sys.exit(1)
scan_file = None
token = nessus_login(NESSUS_URL, NESSUS_USERNAME, NESSUS_PASSWORD)
if not token:
sys.exit(1)
x_token = get_x_api_token(NESSUS_URL, token)
if not x_token:
sys.exit(1)
scan_id = nessus_add_target(
NESSUS_URL,
token,
x_token,
NESSUS_SCAN_TARGET,
NESSUS_SCAN_TEMPLATE,
NESSUS_SCAN_NAME,
)
if not scan_id:
sys.exit(1)
status = nessus_scan_run(NESSUS_URL, scan_id, token, x_token)
if status != "error":
scan_file = nessus_scan_export(NESSUS_URL, scan_id, token, x_token)
if scan_file:
plugin = PluginsManager().get_plugin("nessus")
plugin.parseOutputString(scan_file)
print(plugin.get_json())
else:
print("Scan file was empty", file=sys.stderr)
def main():
my_envs = os.environ
# If the script is run outside the dispatcher
# the environment variables
# are checked.
# ['EXECUTOR_CONFIG_NAME_URL', 'ARACHNI_PATH']
if "EXECUTOR_CONFIG_NAME_URL" in my_envs:
url_analyze = os.environ.get("EXECUTOR_CONFIG_NAME_URL")
url = urlparse(url_analyze)
if url.scheme != "http" and url.scheme != "https":
url_analyze = f"http://{url_analyze}"
else:
print("Param NAME_URL no passed", file=sys.stderr)
sys.exit()
if "ARACHNI_PATH" in my_envs:
path_arachni = os.environ.get("ARACHNI_PATH")
else:
print("Environment variable ARACHNI_PATH no set", file=sys.stderr)
sys.exit()
os.chdir(path_arachni)
file_afr = tempfile.NamedTemporaryFile(mode="w", suffix=".afr")
cmd = ["./arachni", url_analyze, "--report-save-path", file_afr.name]
arachni_command = subprocess.run(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
flush_messages(arachni_command)
name_xml = tempfile.NamedTemporaryFile(mode="w", suffix=".xml")
cmd = [
"./arachni_reporter",
file_afr.name,
"--reporter",
f"xml:outfile={name_xml.name}",
]
arachni_reporter_process = subprocess.run(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
flush_messages(arachni_reporter_process)
plugin = PluginsManager().get_plugin("arachni")
with open(name_xml.name, "r") as f:
plugin.parseOutputString(f.read())
print(plugin.get_json())
name_xml.close()
file_afr.close()
class ReportsManager(Thread):
def __init__(self, upload_reports_queue, *args, **kwargs):
super().__init__(*args, **kwargs)
self.upload_reports_queue = upload_reports_queue
self.plugins_manager = PluginsManager(config.faraday_server.custom_plugins_folder)
self.__event = threading.Event()
def stop(self):
logger.debug("Stop Reports Manager")
self.__event.set()
def send_report_request(self, workspace_name, report_json, user):
logger.info("Send Report data to workspace [%s]", workspace_name)
from faraday.server.web import app # pylint:disable=import-outside-toplevel
with app.app_context():
ws = Workspace.query.filter_by(name=workspace_name).one()
schema = BulkCreateSchema()
data = schema.load(report_json)
data = add_creator(data, user)
bulk_create(ws, data, True)
def process_report(self, workspace, file_path, plugin_id, user):
plugin = self.plugins_manager.get_plugin(plugin_id)
if plugin:
try:
logger.info("Processing report [%s] with plugin [%s]", file_path, plugin.id)
plugin.processReport(file_path)
vulns_data = plugin.get_data()
except Exception as e:
logger.error("Processing Error: %s", e)
logger.exception(e)
else:
try:
self.send_report_request(workspace, vulns_data, user)
logger.info("Report processing finished")
except Exception as e:
logger.exception(e)
logger.error("Save Error: %s", e)
else:
logger.info("No plugin detected for report [%s]", file_path)
def run(self):
logger.debug("Start Reports Manager")
while not self.__event.is_set():
try:
workspace, file_path, plugin_id, user = self.upload_reports_queue.get(False, timeout=0.1)
logger.info("Processing raw report %s", file_path)
if os.path.isfile(file_path):
self.process_report(workspace, file_path, plugin_id, user)
else:
logger.warning("Report file [%s] don't exists", file_path)
except Empty:
self.__event.wait(0.1)
except KeyboardInterrupt:
logger.info("Keyboard interrupt, stopping report processing thread")
self.stop()
except Exception as ex:
logger.exception(ex)
continue
def generate_reports_tests(force, debug):
generated_summaries = 0
analysed_reports = 0
click.echo(f"{colorama.Fore.GREEN}Generate Faraday Plugins Tests Summary")
plugins_manager = PluginsManager()
analyzer = ReportAnalyzer(plugins_manager)
for report_file_path in list_report_files():
if debug:
click.echo(f"File: {report_file_path}")
plugin: PluginBase = analyzer.get_plugin(report_file_path)
if not plugin:
click.echo(
f"{colorama.Fore.YELLOW}Plugin for file: ({report_file_path}) not found"
)
else:
with open(report_file_path, 'rb') as f:
m = hashlib.md5(f.read())
file_checksum = m.hexdigest()
if file_checksum not in REPORTS_CHECKSUM:
REPORTS_CHECKSUM.append(file_checksum)
else:
click.echo(
f"{colorama.Fore.YELLOW}Ignore duplicated file: ({report_file_path})"
)
continue
analysed_reports += 1
report_file_name = os.path.basename(report_file_path)
plugin_name = plugin.id
plugin_path = os.path.join(REPORT_COLLECTION_DIR,
FARADAY_PLUGINS_TESTS_DIR, plugin_name)
if not os.path.isdir(plugin_path):
os.mkdir(plugin_path)
dst_report_file_path = os.path.join(plugin_path, report_file_name)
summary_needed = False
summary_file = f"{os.path.splitext(dst_report_file_path)[0]}_summary.json"
if not os.path.isfile(dst_report_file_path) or force:
summary_needed = True
shutil.copyfile(report_file_path, dst_report_file_path)
if not os.path.isfile(summary_file) or force:
summary_needed = True
if summary_needed:
try:
plugin.processReport(report_file_path)
click.echo(
f"{colorama.Fore.GREEN}Generate Summary for: {dst_report_file_path} [{plugin}]"
)
summary = plugin.get_summary()
with open(summary_file, "w") as f:
json.dump(summary, f)
generated_summaries += 1
except Exception as e:
click.echo(
f"{colorama.Fore.RED}Error generating summary for file: {report_file_path} [{plugin}]: [{e}]"
)
click.echo(
f"Generated {generated_summaries} summaries of {analysed_reports} reports"
)
def test_autodetection_on_all_report_collection(report_filename):
plugins_manager = PluginsManager()
analyzer = ReportAnalyzer(plugins_manager)
plugin: PluginBase = analyzer.get_plugin(report_filename)
assert plugin, report_filename
plugin.processReport(report_filename)
plugin_json = json.loads(plugin.get_json())
assert "hosts" in plugin_json
assert "command" in plugin_json
assert len(plugin_json) == 2
def list_plugins(custom_plugins_folder):
plugins_manager = PluginsManager(custom_plugins_folder)
click.echo(click.style("Available Plugins:", fg="cyan"))
loaded_plugins = 0
for plugin_id, plugin in plugins_manager.get_plugins():
console_enabled = plugin._command_regex is not None
console_enabled_color = "green" if console_enabled else "red"
console_enabled_text = click.style(
f"{'Yes' if console_enabled else 'No'}", fg=console_enabled_color)
report_enabled = isinstance(plugin, PluginByExtension)
report_enabled_color = "green" if report_enabled else "red"
report_enabled_text = click.style(
f"{'Yes' if report_enabled else 'No'}", fg=report_enabled_color)
click.echo(
f"{plugin.id:15} - [Command: {console_enabled_text:>12} - Report: {report_enabled_text:>12}] - {plugin.name} "
)
loaded_plugins += 1
click.echo(click.style(f"Loaded Plugins: {loaded_plugins}", fg="cyan"))
def detect_command(command, custom_plugins_folder):
plugins_manager = PluginsManager(custom_plugins_folder)
analyzer = CommandAnalyzer(plugins_manager)
plugin = analyzer.get_plugin(command)
if plugin:
click.echo(click.style(f"Faraday Plugin: {plugin.id}", fg="cyan"))
else:
click.echo(click.style(f"Failed to detect command: {command}",
fg="red"),
err=True)
def test_detected_tools_on_all_report_collection(report_filename, benchmark):
plugins_manager = PluginsManager()
analyzer = ReportAnalyzer(plugins_manager)
plugin: PluginBase = analyzer.get_plugin(report_filename)
if not plugin:
return
assert plugin, report_filename
benchmark(plugin.processReport, report_filename)
plugin_json = json.loads(plugin.get_json())
assert "hosts" in plugin_json
assert "command" in plugin_json
assert os.path.isfile(report_filename) is True
def detect_report(report_file, custom_plugins_folder):
if not os.path.isfile(report_file):
click.echo(click.style(f"File {report_file} Don't Exists", fg="red"))
else:
plugins_manager = PluginsManager(custom_plugins_folder)
analyzer = ReportAnalyzer(plugins_manager)
plugin = analyzer.get_plugin(report_file)
if plugin:
click.echo(click.style(f"Faraday Plugin: {plugin.id}", fg="cyan"))
else:
click.echo(click.style(f"Failed to detect report: {report_file}",
fg="red"),
err=True)
def main():
""" main function """
with RedirectOutput():
scanner = ness6rest.Scanner(url=NESSUS_URL,
login=NESSUS_USERNAME,
password=NESSUS_PASSWORD,
insecure=True)
scantemplate = os.getenv("NESSUS_SCANTEMPLATE", "basic")
scanner.scan_add(targets=NESSUS_SCANTARGET,
template=scantemplate,
name="Faraday Agent Scan")
print("Starting scan")
scanner.scan_run()
#This blocks execution until the scan stops running
scanner._scan_status()
plugin = PluginsManager().get_plugin("nessus")
plugin.parseOutputString(scanner.download_scan(export_format="nessus"))
print(plugin.get_json())
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.TABLE_PRETTY_FORMAT = "psql"
# hide unwanted settings
settings_to_hide = ["debug"]
for setting_name in settings_to_hide:
self.remove_settable(setting_name)
f = Figlet(font="slant")
intro = []
intro.append(style(f.renderText("Faraday Cli"), fg="red"))
if active_config.faraday_url and active_config.token:
intro.append(
style(f"Server: {active_config.faraday_url}", fg="green"))
self.api_client = FaradayApi(
active_config.faraday_url,
ignore_ssl=active_config.ignore_ssl,
token=active_config.token,
)
else:
self.api_client = FaradayApi()
intro.append(
style(f"Missing faraday server, run 'auth'", fg="yellow"))
self.intro = "\n".join(intro)
self.data_queue = queue.Queue()
self.custom_plugins_path = None
self.update_prompt()
self.add_settable(
Settable(
"custom_plugins_path",
str,
"Path of custom plugins",
onchange_cb=self._onchange_custom_plugins_path,
))
self.plugins_manager = PluginsManager(
active_config.custom_plugins_folder)
self.report_analyzer = ReportAnalyzer(self.plugins_manager)
self.command_analyzer = CommandAnalyzer(self.plugins_manager)
def list_plugins():
plugins_manager = PluginsManager()
for _, plugin in plugins_manager.get_plugins():
click.echo(f"{plugin.id}")
def command(api_client, custom_plugins_folder, plugin_id, workspace,
hide_output, command):
if workspace:
if not api_client.is_workspace_valid(workspace):
click.secho(f"Invalid workspace: {workspace}", fg="red")
return
else:
destination_workspace = workspace
else:
if not active_config.workspace:
click.secho(f"Missing default workspace", fg="red")
return
else:
destination_workspace = active_config.workspace
plugins_manager = PluginsManager(custom_plugins_folder)
analyzer = CommandAnalyzer(plugins_manager)
if plugin_id:
plugin = plugins_manager.get_plugin(plugin_id)
if not plugin:
click.echo(click.style(f"Invalid Plugin: {plugin_id}", fg="red"),
err=True)
return
else:
plugin = analyzer.get_plugin(command)
if not plugin:
click.echo(click.style(f"Failed to detect command: {command}",
fg="red"),
err=True)
return
current_path = os.path.abspath(os.getcwd())
modified_command = plugin.processCommandString(getpass.getuser(),
current_path, command)
if modified_command:
command = modified_command
p = subprocess.Popen(shlex.split(command),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
output = io.StringIO()
while True:
retcode = p.poll()
line = p.stdout.readline().decode('utf-8')
if not hide_output:
sys.stdout.write(line)
output.write(line)
if retcode is not None:
extra_lines = map(lambda x: x.decode('utf-8'),
p.stdout.readlines())
if not hide_output:
sys.stdout.writelines(line)
output.writelines(extra_lines)
break
output_value = output.getvalue()
if retcode == 0:
plugin.processOutput(output_value)
click.secho(
f"Sending data from command {command} to {destination_workspace}",
fg="green")
api_client.bulk_create(destination_workspace, plugin.get_data())
else:
click.secho("Command execution error!!", fg="red")
def _create_plugin_manager(self):
self.plugins_manager = PluginsManager(
self.custom_plugins_path, ignore_info=self.ignore_info_severity)
self.report_analyzer = ReportAnalyzer(self.plugins_manager)
self.command_analyzer = CommandAnalyzer(self.plugins_manager)
def main():
# separate the target list with comma
NUCLEI_TARGET = os.getenv("EXECUTOR_CONFIG_NUCLEI_TARGET")
# separate the exclude list with comma
NUCLEI_EXCLUDE = os.getenv("EXECUTOR_CONFIG_NUCLEI_EXCLUDE", None)
NUCLEI_TEMPLATES = os.getenv("NUCLEI_TEMPLATES")
target_list = NUCLEI_TARGET.split(",")
with tempfile.TemporaryDirectory() as tempdirname:
name_urls = Path(tempdirname) / "urls.txt"
name_output = Path(tempdirname) / "output.json"
if len(target_list) > 1:
with open(name_urls, "w") as f:
for url in target_list:
url_parse = urlparse(url)
if is_ip(url_parse.netloc) or is_ip(url_parse.path):
print(f"Is {url} not valid.", file=sys.stderr)
else:
if not url_parse.scheme:
f.write(f"http://{url}\n")
else:
f.write(f"{url}\n")
cmd = [
"nuclei",
"-l",
name_urls,
"-t",
NUCLEI_TEMPLATES,
]
else:
url_parse = urlparse(NUCLEI_TARGET)
if is_ip(url_parse.hostname) or is_ip(url_parse.path):
print(f"Is {NUCLEI_TARGET} not valid.", file=sys.stderr)
sys.exit()
else:
if not url_parse.scheme:
url = f"http://{NUCLEI_TARGET}"
else:
url = f"{NUCLEI_TARGET}"
cmd = [
"nuclei",
"-target",
url,
"-t",
NUCLEI_TEMPLATES,
]
if NUCLEI_EXCLUDE is not None:
exclude_list = NUCLEI_EXCLUDE.split(",")
for exclude in exclude_list:
cmd += ["-exclude", str(Path(NUCLEI_TEMPLATES) / exclude)]
cmd += ["-json", "-o", name_output]
nuclei_process = subprocess.run(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if len(nuclei_process.stdout) > 0:
print(f"Nuclei stdout: {nuclei_process.stdout.decode('utf-8')}",
file=sys.stderr)
if len(nuclei_process.stderr) > 0:
print(f"Nuclei stderr: {nuclei_process.stderr.decode('utf-8')}",
file=sys.stderr)
plugin = PluginsManager().get_plugin("nuclei")
plugin.parseOutputString(nuclei_process.stdout.decode("utf-8"))
print(plugin.get_json())
def __init__(self, upload_reports_queue, *args, **kwargs):
super().__init__(*args, **kwargs)
self.upload_reports_queue = upload_reports_queue
self.plugins_manager = PluginsManager()
self._must_stop = False
def main():
url_target = os.environ.get("EXECUTOR_CONFIG_W3AF_TARGET_URL")
if not url_target:
print("URL not provided", file=sys.stderr)
sys.exit()
# If the script is run outside the dispatcher the environment variables
# are checked.
# ['W3AF_PATH']
if "W3AF_PATH" in os.environ:
with tempfile.TemporaryDirectory() as tempdirname:
name_result = Path(tempdirname) / "config_report_file.w3af"
with open(name_result, "w") as f:
command_text = ("plugins\n output console,xml_file\n"
" output\n output config xml_file\n "
"set output_file "
f"{tempdirname}/output-w3af.xml\n set verbose"
" True\n back\n output config console\n set "
"verbose False\n back\n crawl all, "
"!bing_spider, !google_spider, !spider_man\n "
"crawl\n grep all\n grep\n audit all\n "
"audit\n bruteforce all\n bruteforce\n "
f"back\n target\n set target {url_target}\n "
f"back\n start\n exit")
f.write(command_text)
try:
os.chdir(path=os.environ.get("W3AF_PATH"))
except FileNotFoundError:
print(
"The directory where w3af is located could not be found. "
"Check environment variable W3AF_PATH = "
f'{os.environ.get("W3AF_PATH")}',
file=sys.stderr,
)
sys.exit()
if os.path.isfile("w3af_console"):
cmd = [
"./w3af_console",
"-s",
name_result,
]
w3af_process = subprocess.run(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if len(w3af_process.stdout) > 0:
print(
f"W3AF stdout: {w3af_process.stdout.decode('utf-8')}",
file=sys.stderr,
)
if len(w3af_process.stderr) > 0:
print(
f"W3AF stderr: {w3af_process.stderr.decode('utf-8')}",
file=sys.stderr,
)
plugin = PluginsManager().get_plugin("w3af")
plugin.parseOutputString(f"{tempdirname}/output-w3af.xml")
print(plugin.get_json())
else:
print(
"w3af_console file could not be found. For this reason the"
" command cannot be run. Actual value = "
f'{os.environ.get("W3AF_PATH")}/w3af_console ',
file=sys.stderr,
)
sys.exit()
else:
print("W3AF_PATH not set", file=sys.stderr)
sys.exit()
from flask_wtf.csrf import validate_csrf
from werkzeug.utils import secure_filename
from wtforms import ValidationError
from faraday.server.utils.web import gzipped
from faraday.server.models import Workspace, Command, db
from faraday.server import config
from faraday_plugins.plugins.manager import PluginsManager, ReportAnalyzer
upload_api = Blueprint('upload_reports', __name__)
logger = logging.getLogger(__name__)
plugins_manager = PluginsManager(config.faraday_server.custom_plugins_folder)
report_analyzer = ReportAnalyzer(plugins_manager)
@gzipped
@upload_api.route('/v2/ws/<workspace>/upload_report', methods=['POST'])
def file_upload(workspace=None):
"""
Upload a report file to Server and process that report with Faraday client plugins.
"""
logger.info("Importing new plugin report in server...")
# Authorization code copy-pasted from server/api/base.py
ws = Workspace.query.filter_by(name=workspace).first()
if not ws or not ws.active:
# Don't raise a 403 to prevent workspace name enumeration
abort(404, f"Workspace disabled: {workspace}")
def file_upload(self, workspace_name=None):
"""
---
post:
tags: ["Workspace", "File"]
description: Upload a report file to create data within the given workspace
responses:
201:
description: Created
400:
description: Bad request
403:
description: Forbidden
tags: ["Workspace", "File"]
responses:
200:
description: Ok
"""
logger.info("Importing new plugin report in server...")
# Authorization code copy-pasted from server/api/base.py
ws = Workspace.query.filter_by(name=workspace_name).first()
if not ws or not ws.active:
# Don't raise a 403 to prevent workspace name enumeration
abort(404, f"Workspace disabled: {workspace_name}")
if 'file' not in request.files:
abort(400)
try:
validate_csrf(request.form.get('csrf_token'))
except ValidationError:
abort(403)
report_file = request.files['file']
if report_file:
chars = string.ascii_uppercase + string.digits
random_prefix = ''.join(random.choice(chars)
for x in range(12)) # nosec
raw_report_filename = f'{random_prefix}_{secure_filename(report_file.filename)}'
try:
file_path = CONST_FARADAY_HOME_PATH / 'uploaded_reports' \
/ raw_report_filename
with file_path.open('wb') as output:
output.write(report_file.read())
except AttributeError:
logger.warning(
"Upload reports in WEB-UI not configurated, run Faraday client and try again..."
)
abort(
make_response(
jsonify(
message=
"Upload reports not configurated: Run faraday client and start Faraday server again"
), 500))
else:
logger.info(f"Get plugin for file: {file_path}")
plugins_manager = PluginsManager(
ReportsSettings.settings.custom_plugins_folder)
report_analyzer = ReportAnalyzer(plugins_manager)
plugin = report_analyzer.get_plugin(file_path)
if not plugin:
logger.info("Could not get plugin for file")
abort(
make_response(jsonify(message="Invalid report file"),
400))
else:
logger.info(
f"Plugin for file: {file_path} Plugin: {plugin.id}")
workspace_instance = Workspace.query.filter_by(
name=workspace_name).one()
command = Command()
command.workspace = workspace_instance
command.start_date = datetime.utcnow()
command.import_source = 'report'
# The data will be updated in the bulk_create function
command.tool = "In progress"
command.command = "In progress"
db.session.add(command)
db.session.commit()
REPORTS_QUEUE.put(
(workspace_instance.name, command.id, file_path,
plugin.id, flask_login.current_user.id))
return make_response(
jsonify(message="ok", command_id=command.id), 200)
else:
abort(make_response(jsonify(message="Missing report file"), 400))
class ReportsManager(Thread):
def __init__(self, upload_reports_queue, *args, **kwargs):
super().__init__(*args, **kwargs)
self.upload_reports_queue = upload_reports_queue
self.plugins_manager = PluginsManager(
config.faraday_server.custom_plugins_folder)
self.__event = threading.Event()
def stop(self):
logger.debug("Stop Reports Manager")
self.__event.set()
def send_report_request(self, workspace_name: str, command_id: int,
report_json: dict, user_id: int):
logger.info("Send Report data to workspace [%s]", workspace_name)
from faraday.server.web import app # pylint:disable=import-outside-toplevel
with app.app_context():
ws = Workspace.query.filter_by(name=workspace_name).one()
command = Command.query.filter_by(id=command_id).one()
user = User.query.filter_by(id=user_id).one()
schema = BulkCreateSchema()
data = schema.load(report_json)
data = add_creator(data, user)
bulk_create(ws, command, data, True, True)
def process_report(self, workspace_name: str, command_id: int,
file_path: Path, plugin_id: int, user_id: int):
plugin = self.plugins_manager.get_plugin(plugin_id)
if plugin:
try:
logger.info(f"Processing report [{file_path}] with plugin ["
f"{plugin.id}")
plugin.processReport(str(file_path))
vulns_data = plugin.get_data()
del vulns_data['command']['duration']
except Exception as e:
logger.error("Processing Error: %s", e)
logger.exception(e)
else:
try:
self.send_report_request(workspace_name, command_id,
vulns_data, user_id)
logger.info("Report processing finished")
except Exception as e:
logger.exception(e)
logger.error("Save Error: %s", e)
else:
logger.info(f"No plugin detected for report [{file_path}]")
def run(self):
logger.debug("Start Reports Manager")
while not self.__event.is_set():
try:
tpl: Tuple[str, int, Path, int, int] = \
self.upload_reports_queue.get(False, timeout=0.1)
workspace_name, command_id, file_path, plugin_id, user_id = tpl
logger.info(f"Processing raw report {file_path}")
if file_path.is_file():
self.process_report(workspace_name, command_id, file_path,
plugin_id, user_id)
else:
logger.warning(f"Report file [{file_path}] don't exists",
file_path)
except Empty:
self.__event.wait(0.1)
except KeyboardInterrupt:
logger.info(
"Keyboard interrupt, stopping report processing thread")
self.stop()
except Exception as ex:
logger.exception(ex)
continue
import json
import pytest
from faraday_plugins.plugins.manager import PluginsManager, CommandAnalyzer
from faraday_plugins.plugins.plugin import PluginBase
plugins_manager = PluginsManager()
analyzer = CommandAnalyzer(plugins_manager)
COMMANDS_FILE = './tests/commands.json'
def list_commands():
with open(COMMANDS_FILE) as f:
commands_dict = json.load(f)
for command_data in commands_dict["commands"]:
yield command_data
@pytest.mark.parametrize("command_data", list_commands())
def test_autodetected_on_commands(command_data):
plugin_id = command_data["plugin_id"]
command_string = command_data["command"]
command_result = command_data["command_result"]
plugin: PluginBase = analyzer.get_plugin(command_string)
assert plugin, command_string
assert plugin.id.lower() == plugin_id.lower()
assert plugin.command.lower() == command_result.lower()