def process_report(report_file, plugin_id, custom_plugins_folder, summary,
output_file, ignore_info):
if not os.path.isfile(report_file):
click.echo(click.style(f"File {report_file} Don't Exists", fg="red"),
err=True)
else:
plugins_manager = PluginsManager(custom_plugins_folder,
ignore_info=ignore_info)
analyzer = ReportAnalyzer(plugins_manager)
if plugin_id:
plugin = plugins_manager.get_plugin(plugin_id)
if not plugin:
click.echo(click.style(f"Invalid Plugin: {plugin_id}",
fg="red"),
err=True)
return
else:
plugin = analyzer.get_plugin(report_file)
if not plugin:
click.echo(click.style(
f"Failed to detect report: {report_file}", fg="red"),
err=True)
return
plugin.processReport(Path(report_file), getpass.getuser())
if summary:
click.echo(json.dumps(plugin.get_summary(), indent=4))
else:
if output_file:
with open(output_file, "w") as f:
json.dump(plugin.get_data(), f)
else:
click.echo(json.dumps(plugin.get_data(), indent=4))
class ReportsManager(Thread):
def __init__(self, upload_reports_queue, *args, **kwargs):
super().__init__(*args, **kwargs)
self.upload_reports_queue = upload_reports_queue
self.plugins_manager = PluginsManager(config.faraday_server.custom_plugins_folder)
self.__event = threading.Event()
def stop(self):
logger.debug("Stop Reports Manager")
self.__event.set()
def send_report_request(self, workspace_name, report_json, user):
logger.info("Send Report data to workspace [%s]", workspace_name)
from faraday.server.web import app # pylint:disable=import-outside-toplevel
with app.app_context():
ws = Workspace.query.filter_by(name=workspace_name).one()
schema = BulkCreateSchema()
data = schema.load(report_json)
data = add_creator(data, user)
bulk_create(ws, data, True)
def process_report(self, workspace, file_path, plugin_id, user):
plugin = self.plugins_manager.get_plugin(plugin_id)
if plugin:
try:
logger.info("Processing report [%s] with plugin [%s]", file_path, plugin.id)
plugin.processReport(file_path)
vulns_data = plugin.get_data()
except Exception as e:
logger.error("Processing Error: %s", e)
logger.exception(e)
else:
try:
self.send_report_request(workspace, vulns_data, user)
logger.info("Report processing finished")
except Exception as e:
logger.exception(e)
logger.error("Save Error: %s", e)
else:
logger.info("No plugin detected for report [%s]", file_path)
def run(self):
logger.debug("Start Reports Manager")
while not self.__event.is_set():
try:
workspace, file_path, plugin_id, user = self.upload_reports_queue.get(False, timeout=0.1)
logger.info("Processing raw report %s", file_path)
if os.path.isfile(file_path):
self.process_report(workspace, file_path, plugin_id, user)
else:
logger.warning("Report file [%s] don't exists", file_path)
except Empty:
self.__event.wait(0.1)
except KeyboardInterrupt:
logger.info("Keyboard interrupt, stopping report processing thread")
self.stop()
except Exception as ex:
logger.exception(ex)
continue
def main():
my_envs = os.environ
# If the script is run outside the dispatcher
# the environment variables
# are checked.
tool = os.environ.get("EXECUTOR_CONFIG_TOOL", None)
if "EXECUTOR_CONFIG_REPORT_NAME" in my_envs:
report_name = os.environ.get("EXECUTOR_CONFIG_REPORT_NAME")
else:
print("Argument REPORT_NAME no set", file=sys.stderr)
sys.exit()
if "REPORTS_PATH" in my_envs:
report_dir = os.environ.get("REPORTS_PATH")
else:
print("Environment variable REPORT_DIR no set", file=sys.stderr)
sys.exit()
filepath = Path(report_dir) / report_name
manager = PluginsManager()
if tool is not None:
plugin = manager.get_plugin(tool)
else:
plugin = ReportAnalyzer(manager).get_plugin(filepath)
print(f"Detector found it as an {plugin} report", file=sys.stderr)
print(f"Parsing {filepath} report", file=sys.stderr)
plugin.processReport(str(filepath))
print(plugin.get_json())
def process_command(command, plugin_id, custom_plugins_folder, dont_run,
summary, output_file, show_output, ignore_info):
plugins_manager = PluginsManager(custom_plugins_folder,
ignore_info=ignore_info)
analyzer = CommandAnalyzer(plugins_manager)
if plugin_id:
plugin = plugins_manager.get_plugin(plugin_id)
if not plugin:
click.echo(click.style(f"Invalid Plugin: {plugin_id}", fg="red"),
err=True)
return
else:
plugin = analyzer.get_plugin(command)
if not plugin:
click.echo(click.style(f"Failed to detect command: {command}",
fg="red"),
err=True)
return
current_path = os.path.abspath(os.getcwd())
modified_command = plugin.processCommandString(getpass.getuser(),
current_path, command)
if modified_command:
command = modified_command
if dont_run:
color_message = click.style("Command: ", fg="green")
click.echo(f"{color_message} {command}")
else:
p = subprocess.Popen(shlex.split(command),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
output = io.StringIO()
while True:
retcode = p.poll()
line = p.stdout.readline().decode('utf-8')
if show_output:
sys.stdout.write(line)
output.write(line)
if retcode is not None:
extra_lines = map(lambda x: x.decode('utf-8'),
p.stdout.readlines())
if show_output:
sys.stdout.writelines(line)
output.writelines(extra_lines)
break
output_value = output.getvalue()
if retcode == 0:
plugin.processOutput(output_value)
if summary:
click.echo(json.dumps(plugin.get_summary(), indent=4))
else:
if output_file:
with open(output_file, "w") as f:
json.dump(plugin.get_data(), f)
else:
click.echo(json.dumps(plugin.get_data(), indent=4))
else:
click.echo(click.style("Command execution error!!", fg="red"),
err=True)
def process_report(workspace_name: str, command_id: int, file_path: Path,
plugin_id: Optional[int], user_id: Optional[int]):
from faraday.server.web import get_app # pylint:disable=import-outside-toplevel
with get_app().app_context():
if plugin_id is not None:
plugins_manager = PluginsManager(
ReportsSettings.settings.custom_plugins_folder,
ignore_info=ReportsSettings.settings.ignore_info_severity)
logger.info(
f"Reports Manager: [Custom plugins folder: "
f"[{ReportsSettings.settings.custom_plugins_folder}]"
f"[Ignore info severity: {ReportsSettings.settings.ignore_info_severity}]"
)
plugin = plugins_manager.get_plugin(plugin_id)
if plugin:
try:
logger.info(
f"Processing report [{file_path}] with plugin ["
f"{plugin.id}]")
plugin.processReport(str(file_path))
vulns_data = plugin.get_data()
del vulns_data['command']['duration']
except Exception as e:
logger.error("Processing Error: %s", e)
logger.exception(e)
return
else:
logger.error(f"No plugin detected for report [{file_path}]")
return
else:
try:
with file_path.open("r") as f:
vulns_data = json.load(f)
except Exception as e:
logger.error("Loading data from json file: %s [%s]", file_path,
e)
logger.exception(e)
return
if plugin_id is None:
logger.debug("Removing file: %s", file_path)
os.remove(file_path)
else:
if faraday_server.delete_report_after_process:
os.remove(file_path)
set_end_date = True
try:
send_report_data(workspace_name, command_id, vulns_data, user_id,
set_end_date)
logger.info("Report processing finished")
except Exception as e:
logger.exception(e)
logger.error("Save Error: %s", e)
class ReportsManager(Thread):
def __init__(self, upload_reports_queue, *args, **kwargs):
super().__init__(*args, **kwargs)
self.upload_reports_queue = upload_reports_queue
self.plugins_manager = PluginsManager(
config.faraday_server.custom_plugins_folder)
self.__event = threading.Event()
def stop(self):
logger.debug("Stop Reports Manager")
self.__event.set()
def send_report_request(self, workspace_name: str, command_id: int,
report_json: dict, user_id: int):
logger.info("Send Report data to workspace [%s]", workspace_name)
from faraday.server.web import app # pylint:disable=import-outside-toplevel
with app.app_context():
ws = Workspace.query.filter_by(name=workspace_name).one()
command = Command.query.filter_by(id=command_id).one()
user = User.query.filter_by(id=user_id).one()
schema = BulkCreateSchema()
data = schema.load(report_json)
data = add_creator(data, user)
bulk_create(ws, command, data, True, True)
def process_report(self, workspace_name: str, command_id: int,
file_path: Path, plugin_id: int, user_id: int):
plugin = self.plugins_manager.get_plugin(plugin_id)
if plugin:
try:
logger.info(f"Processing report [{file_path}] with plugin ["
f"{plugin.id}")
plugin.processReport(str(file_path))
vulns_data = plugin.get_data()
del vulns_data['command']['duration']
except Exception as e:
logger.error("Processing Error: %s", e)
logger.exception(e)
else:
try:
self.send_report_request(workspace_name, command_id,
vulns_data, user_id)
logger.info("Report processing finished")
except Exception as e:
logger.exception(e)
logger.error("Save Error: %s", e)
else:
logger.info(f"No plugin detected for report [{file_path}]")
def run(self):
logger.debug("Start Reports Manager")
while not self.__event.is_set():
try:
tpl: Tuple[str, int, Path, int, int] = \
self.upload_reports_queue.get(False, timeout=0.1)
workspace_name, command_id, file_path, plugin_id, user_id = tpl
logger.info(f"Processing raw report {file_path}")
if file_path.is_file():
self.process_report(workspace_name, command_id, file_path,
plugin_id, user_id)
else:
logger.warning(f"Report file [{file_path}] don't exists",
file_path)
except Empty:
self.__event.wait(0.1)
except KeyboardInterrupt:
logger.info(
"Keyboard interrupt, stopping report processing thread")
self.stop()
except Exception as ex:
logger.exception(ex)
continue
def command(api_client, custom_plugins_folder, plugin_id, workspace,
hide_output, command):
if workspace:
if not api_client.is_workspace_valid(workspace):
click.secho(f"Invalid workspace: {workspace}", fg="red")
return
else:
destination_workspace = workspace
else:
if not active_config.workspace:
click.secho(f"Missing default workspace", fg="red")
return
else:
destination_workspace = active_config.workspace
plugins_manager = PluginsManager(custom_plugins_folder)
analyzer = CommandAnalyzer(plugins_manager)
if plugin_id:
plugin = plugins_manager.get_plugin(plugin_id)
if not plugin:
click.echo(click.style(f"Invalid Plugin: {plugin_id}", fg="red"),
err=True)
return
else:
plugin = analyzer.get_plugin(command)
if not plugin:
click.echo(click.style(f"Failed to detect command: {command}",
fg="red"),
err=True)
return
current_path = os.path.abspath(os.getcwd())
modified_command = plugin.processCommandString(getpass.getuser(),
current_path, command)
if modified_command:
command = modified_command
p = subprocess.Popen(shlex.split(command),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
output = io.StringIO()
while True:
retcode = p.poll()
line = p.stdout.readline().decode('utf-8')
if not hide_output:
sys.stdout.write(line)
output.write(line)
if retcode is not None:
extra_lines = map(lambda x: x.decode('utf-8'),
p.stdout.readlines())
if not hide_output:
sys.stdout.writelines(line)
output.writelines(extra_lines)
break
output_value = output.getvalue()
if retcode == 0:
plugin.processOutput(output_value)
click.secho(
f"Sending data from command {command} to {destination_workspace}",
fg="green")
api_client.bulk_create(destination_workspace, plugin.get_data())
else:
click.secho("Command execution error!!", fg="red")