def add_configuration_import(new_export_data, res_client):
"""
Makes a REST request to add a configuration import.
After the request is made, the configuration import is set at a pending state and needs to be confirmed.
If the configuration state is not reported as pending, raise an SDK Exception.
:param new_export_data: A dict representing a configuration import DTO
:type result: dict
:param import_id: The ID of the configuration import to confirm
:type import_id: int
:param res_client: An instantiated res_client for making REST calls
:type res_client: SimpleClient()
:raises SDKException: If the confirmation request fails raise an SDKException
"""
try:
result = res_client.post(IMPORT_URL, new_export_data)
except requests.RequestException as upload_exception:
LOG.debug(new_export_data)
raise SDKException(upload_exception)
else:
assert isinstance(result, dict)
if result.get("status", '') == "PENDING":
confirm_configuration_import(result, result.get("id"), res_client)
else:
raise SDKException(
"Could not import because the server did not return an import ID")
def get_apikey_permissions(path):
"""
Returns a list of api keys to allow an integration to run.
:param path: Location to file with api keys one per line.
:return apikey_permissions: Return list of api keys.
"""
try:
# Read the apikey_permissions.txt file into a List
apikey_permissions_lines = sdk_helpers.read_file(path)
except Exception as err:
raise SDKException(u"Failed to parse configs from apikey_permissions.txt file\nThe apikey_permissions.txt file may "
u"be corrupt. Visit the App Exchange to contact the developer\nReason: {0}".format(err))
# Raise an error if nothing found in the file
if not apikey_permissions_lines:
raise SDKException(u"No content found in provided apikey_permissions.txt file: {0}".format(path))
# Get permissions. Ignore comments where 1st non-whitespace character is a '#'.
apikey_permissions = [p.strip() for p in apikey_permissions_lines if not p.lstrip().startswith("#")]
# Do basic check on api keys to see if they are in correct format.
for p in apikey_permissions:
if not re.match("[_a-zA-Z]*$", p):
raise SDKException(u"Value '{0}' in file '{1}' is not a valid api key value.".format(p, path))
# Ensure that the permissions includes at minimum the set of base permissions.
if not all(p in apikey_permissions for p in BASE_PERMISSIONS):
raise SDKException(u"'The file '{0}' is missing one of the base api key permissions.".format(path))
return apikey_permissions
def execute_command(self, args):
LOG.debug("called: CmdCodegen.execute_command()")
if args.gather_results:
if not args.package:
raise SDKException("'-p' must be specified when using '{0}'".format(constants.SUB_CMD_OPT_GATHER_RESULTS))
SDKException.command_ran = "{0} {1}".format(self.CMD_NAME, constants.SUB_CMD_OPT_GATHER_RESULTS)
self._get_results_from_log_file(args)
elif args.reload:
if not args.package:
raise SDKException("'-p' must be specified when using '--reload'")
SDKException.command_ran = "{0} {1}".format(self.CMD_NAME, "--reload")
self._reload_package(args)
elif args.package:
SDKException.command_ran = "{0} {1}".format(self.CMD_NAME, "--package | -p")
self._gen_package(args)
elif not args.package and args.function:
SDKException.command_ran = "{0} {1}".format(self.CMD_NAME, "--function | -f")
self._gen_function(args)
else:
self.parser.print_help()
def get_icon(icon_name, path_to_icon, width_accepted, height_accepted, default_path_to_icon):
"""
TODO: update docstring with correct standard
Returns the icon at path_to_icon as a base64 encoded string if it is a valid .png file with the resolution
width_accepted x height_accepted. If path_to_icon does not exist, default_path_to_icon is returned as a base64
encoded string
"""
path_icon_to_use = path_to_icon
# Use default_path_to_icon if path_to_icon does not exist
if not path_icon_to_use or not os.path.isfile(path_icon_to_use):
LOG.warning("WARNING: Default icon will be used\nProvided custom icon path for %s is invalid: %s\nNOTE: %s should be placed in the /icons directory", icon_name, path_icon_to_use, icon_name)
path_icon_to_use = default_path_to_icon
# Validate path_icon_to_use and ensure we have READ permissions
try:
sdk_helpers.validate_file_paths(os.R_OK, path_icon_to_use)
except SDKException as err:
raise OSError("Could not find valid icon file. Looked at two locations:\n{0}\n{1}\n{2}".format(path_to_icon, default_path_to_icon, err.message))
# Get the extension of the file. os.path.splitext returns a Tuple with the file extension at position 1 and can be an empty string
split_path = os.path.splitext(path_icon_to_use)
file_extension = split_path[1]
if not file_extension:
raise SDKException("Provided icon file does not have an extension. Icon file must be .png\nIcon File: {0}".format(path_icon_to_use))
elif file_extension != ".png":
raise SDKException("{0} is not a supported icon file type. Icon file must be .png\nIcon File: {1}".format(file_extension, path_icon_to_use))
# Open the icon_file in Bytes mode to validate its resolution
with open(path_icon_to_use, mode="rb") as icon_file:
# According to: https://en.wikipedia.org/wiki/Portable_Network_Graphics#File_format
# First need to seek 16 bytes:
# 8 bytes: png signature
# 4 bytes: IDHR Chunk Length
# 4 bytes: IDHR Chunk type
icon_file.seek(16)
try:
# Bytes 17-20 = image width. Use struct to unpack big-endian encoded unsigned int
icon_width = struct.unpack(">I", icon_file.read(4))[0]
# Bytes 21-24 = image height. Use struct to unpack big-endian encoded unsigned int
icon_height = struct.unpack(">I", icon_file.read(4))[0]
except Exception as err:
raise SDKException("Failed to read icon's resolution. Icon file corrupt. Icon file must be .png\nIcon File: {0}".format(path_icon_to_use))
# Raise exception if resolution is not accepted
if icon_width != width_accepted or icon_height != height_accepted:
raise SDKException("Icon resolution is {0}x{1}. Resolution must be {2}x{3}\nIcon File: {4}".format(icon_width, icon_height, width_accepted, height_accepted, path_icon_to_use))
# If we get here all validations have passed. Open the file in Bytes mode and encode it as base64 and decode to a utf-8 string
with open(path_icon_to_use, "rb") as icon_file:
encoded_icon_string = base64.b64encode(icon_file.read()).decode("utf-8")
return encoded_icon_string
def get_import_definition_from_customize_py(path_customize_py_file):
"""
Return the ImportDefinition in a customize.py or /util/data/export.res file as a Dictionary.
:param path_customize_py_file: Path to the customize.py file
:return import definition dict from /util/data/export.res if the file exists. Otherwise,
get it from customize.py
"""
# If there is a /util/data/export.res then get the import definition from there.
path_src = os.path.dirname(path_customize_py_file)
path_local_export_res = os.path.join(path_src, PATH_LOCAL_EXPORT_RES)
if os.path.isfile(path_local_export_res):
import_definition = get_import_definition_from_local_export_res(
path_local_export_res)
return import_definition
customize_py = load_customize_py_module(path_customize_py_file)
# Call customization_data() to get all ImportDefinitions that are "yielded"
customize_py_import_definitions_generator = customize_py.customization_data(
)
customize_py_import_definitions = []
# customization_data() returns a Generator object with all yield statements, so we loop them
for definition in customize_py_import_definitions_generator:
if isinstance(definition, ImportDefinition):
customize_py_import_definitions.append(
json.loads(base64.b64decode(definition.value)))
else:
LOG.warning(
"WARNING: Unsupported data found in customize.py file. Expected an ImportDefinition. Got: '%s'",
definition)
# If no ImportDefinition found
if not customize_py_import_definitions:
raise SDKException(
"No ImportDefinition found in the customize.py file")
# If more than 1 found
elif len(customize_py_import_definitions) > 1:
raise SDKException(
"Multiple ImportDefinitions found in the customize.py file. There must only be 1 ImportDefinition defined"
)
# Get the import defintion as dict
customize_py_import_definition = customize_py_import_definitions[0]
# Remove the incident type that was added by codegen that allows the data to import
customize_py_import_definition = remove_default_incident_type_from_import_definition(
customize_py_import_definition)
return customize_py_import_definition
def get_configuration_py_file_path(file_type, setup_py_attributes):
""" Get the location of configuration file config or customize for a package.
If file_type == "customize" check that entry point 'resilient.circuits.apphost.customize' (SUPPORTED_EP[0])
is defined in setup.py of the package.
If file_type == "config" check that entry point 'resilient.circuits.apphost.configsection' (SUPPORTED_EP[1]) is
defined in setup.py of the package, else check 'resilient.circuits.configsection' (SUPPORTED_EP[2]) was detected in
setup.py of the package.
Note: For some packages neither of these files may exist not exist.
:param file_type: File whose location is required should be 'customize' or 'config'.
:param setup_py_attributes: Parsed setup.py content.
:return path_py_file: The customize or config file location for the package.
"""
path_py_file = None
if file_type == "customize":
if SUPPORTED_EP[0] in setup_py_attributes["entry_points"]:
path_py_file = setup_py_attributes["entry_points"][SUPPORTED_EP[0]]
elif file_type == "config":
for ep in SUPPORTED_EP[1:]:
if ep in setup_py_attributes["entry_points"]:
path_py_file = setup_py_attributes["entry_points"][ep]
break
else:
raise SDKException("Unknown option '{}'.".format(file_type))
if path_py_file:
# If configuration file defined in setup.py but does not exist raise an error.
try:
sdk_helpers.validate_file_paths(os.R_OK, path_py_file)
except SDKException:
LOG.info(
"Configuration File '%s' defined as an entry point in 'setup.py' not found at location '%s'.",
file_type, path_py_file)
if not sdk_helpers.validate_file_paths(os.R_OK, path_py_file):
raise SDKException(
"Configuration File '{0}' defined as an entry point in 'setup.py' not found at "
"location '{1}'.".format(file_type, path_py_file))
else:
# For certain packages or threat-feeds these files may not exist.
# Warn user if file not found.
LOG.warning(
"WARNING: Configuration File of type '%s' not defined in 'setup.py'. Ignoring and continuing.",
file_type)
return path_py_file
def parse_file_paths_from_readme(readme_line_list):
"""
Takes a list of strings and looks through to find the links characters in markdown:
The method will raise an SDKException if there is a link started without the provided parenthetical
link in correct syntax.
If no exception is raised, it returns a list of filepaths for linked images in the readme.
:param readme_line_list: list of readme lines that will have the comment lines removed
:type readme_line_list: list[str]
:raises: SDKException if link given in invalid syntax
:return: list of paths to linked files
:rtype: list[str]
"""
paths = []
# first, filter out any comments as those might contain invalid screenshot paths
readme_line_list = re.sub(r"(<!--.*?-->)", "", "".join(readme_line_list), flags=re.DOTALL).splitlines()
# loop through the lines of the readme
for line in readme_line_list:
line = line.strip() # strip any leading and trailing whitespace
# check if the line starts with the Markdown syntax for a linked file
if line.startswith("!["):
if "(" in line and ")" in line:
# look to find the linked file between ( )
paths.append(line[line.rfind("(")+1:line.rfind(")")])
else:
# incorrect syntax for line starting with "![" but not including "( ... )"
raise SDKException(u"Line '{0}' in README has invalid link syntax".format(line))
return paths
def confirm_configuration_import(result, import_id, res_client):
"""
Makes a REST request to confirm a pending configuration import as accepted.
Takes 3 params
The result of a configuration import request
The ID of the configuration import request
A res_client to perform the request
:param result: Result of the configuration import request
:type result: dict
:param import_id: The ID of the configuration import to confirm
:type import_id: int
:param res_client: An instantiated res_client for making REST calls
:type res_client: SimpleClient()
:raises SDKException: If the confirmation request fails raise an SDKException
"""
result["status"] = "ACCEPTED" # Have to confirm changes
uri = "{}/{}".format(IMPORT_URL, import_id)
try:
res_client.put(uri, result)
LOG.info(
"Imported configuration changes successfully to the Resilient Appliance"
)
except requests.RequestException as import_exception:
raise SDKException(repr(import_exception))
def get_workflow_functions(workflow, function_uuid=None):
"""Parses the XML of the Workflow Object and returns
a List of all Functions found. If function_uuid is defined
returns all occurrences of that function.
A Workflow Function can have the attributes:
- uuid: String
- inputs: Dict
- post_processing_script: String
- pre_processing_script: String
- result_name: String"""
return_functions = []
# Workflow XML text
wf_xml = workflow.get("content", {}).get("xml", None)
if wf_xml is None:
raise SDKException(
"Could not load xml content from Workflow: {0}".format(workflow))
# Get the root element + endode in utf8 in order to handle Unicode
root = ET.fromstring(wf_xml.encode("utf8"))
# Get the prefix for each element's tag
tag_prefix = root.tag.replace("definitions", "")
xml_path = "./{0}process/{0}serviceTask/{0}extensionElements/*".format(
tag_prefix)
the_function_elements = []
if function_uuid is not None:
xml_path = "{0}[@uuid='{1}']".format(xml_path, function_uuid)
# Get all elements at xml_path that have the uuid of the function
the_function_elements = root.findall(xml_path)
else:
the_extension_elements = root.findall(xml_path)
for extension_element in the_extension_elements:
if "function" in extension_element.tag:
the_function_elements.append(extension_element)
# Foreach element found, load it as a dictionary and append to return list
for fn_element in the_function_elements:
return_function = json.loads(fn_element.text)
return_function["uuid"] = fn_element.attrib.get("uuid", "")
return_function["result_name"] = return_function.get(
"result_name", None)
return_function["post_processing_script"] = return_function.get(
"post_processing_script", None)
return_function["pre_processing_script"] = return_function.get(
"pre_processing_script", None)
return_functions.append(return_function)
return return_functions
def read_zip_file(path, pattern):
"""Returns unzipped contents of file whose name matches a pattern
in zip file at path.
:param path: Path to zip file.
:param pattern: File pattern to match in the zip file.
:return: file_content: Return unzipped file content.
"""
file_content = None
try:
with ZipFile((path), 'r') as zobj:
# Get all file names matching 'pattern'.
file_matches = [
f for f in zobj.namelist() if pattern.lower() in f.lower()
]
if len(file_matches):
if len(file_matches) > 1:
raise SDKException(
"More than one file matching pattern {0} found in zip file: {1}"
.format(pattern, path))
else:
file_name = file_matches.pop()
# Extract the file.
f = zobj.open(file_name)
# Read file and convert content from bytes to string.
file_content = f.read().decode('utf8', 'ignore')
else:
raise SDKException(
"A file matching pattern {0} was not found in zip file: {1}"
.format(pattern, path))
except BadZipfile:
raise SDKException("Bad zip file {0}.".format(path))
except SDKException as err:
raise err
except Exception as err:
# An an unexpected error trying to read a zipfile.
raise SDKException(
"Got an error '{0}' attempting to read zip file {1}".format(
err, path))
return file_content
def add_tag(tag_name, list_of_objs):
"""
Returns list_of_objs with tag_name added to each object.
Replaces any tags that were there originally to address bug INT-3077
:param tag_name: The name of the tag to add
:param list_of_objs: A list of all the objects you want to add the tag too
:raise: SDKException: if list_of_objs is corrupt
:return: list_of_objs with tag_name added to each object
:rtype: list of dicts
"""
# Create tag_to_add
tag_to_add = {"tag_handle": tag_name, "value": None}
err_msg = "Error adding tag '{0}'. '{1}' (printed above) is not a {2}. Instead a {3} was provided.\nProvided ImportDefinition in the customize.py file may be corrupt"
# Check list_of_objs is a List
if not isinstance(list_of_objs, list):
LOG.error("Error adding tag.\n'list_of_objs': %s", list_of_objs)
raise SDKException(
err_msg.format(tag_name, "list_of_objs", "List",
type(list_of_objs)))
# Loop each object in the List
for obj in list_of_objs:
# If its not a dict, error
if not isinstance(obj, dict):
LOG.error("Error adding tag.\n'list_of_objs': %s\n'obj': %s",
list_of_objs, obj)
raise SDKException(
err_msg.format(tag_name, "obj", "Dictionary", type(obj)))
# Set the obj's 'tags' value to tag_to_add
obj["tags"] = [tag_to_add]
# Return the updated list_of_objs
return list_of_objs
def execute_command(self, args):
LOG.debug("called: CmdDev.execute_command()")
if args.set_version:
if not args.package:
raise SDKException(
"'-p' must be specified when using '--set-version'")
SDKException.command_ran = "{0} {1}".format(
self.CMD_NAME, "--set-version")
self._set_version(args)
else:
self.parser.print_help()
def validate_file_paths(permissions, *args):
"""
Check the given *args paths exist and has the given permissions, else raises an Exception
"""
# For each *args
for path_to_file in args:
# Check the file exists
if not os.path.isfile(path_to_file):
raise SDKException("Could not find file: {0}".format(path_to_file))
if permissions:
# Check we have the correct permissions
has_permissions(permissions, path_to_file)
def validate_dir_paths(permissions, *args):
"""
Check the given *args paths are Directories and have the given permissions, else raises an Exception
"""
# For each *args
for path_to_dir in args:
# Check the dir exists
if not os.path.isdir(path_to_dir):
raise SDKException(
"Could not find directory: {0}".format(path_to_dir))
if permissions:
# Check we have the correct permissions
has_permissions(permissions, path_to_dir)
def get_required_python_version(python_requires_str):
"""
Given a value from the 'python_requires' attribute of setup.py, parse out the
numerical value given for the version required.
:param python_requires_str: str representation of the value assosciated with the 'python_requires' attr in setup.py
:raise SDKException: if format of python_requires is not correct (i.e. in '>=<version>' format)
:return: return the minimum required python version or None if not found
:rtype: tuple with (<major>, <minor>) version format
"""
try:
version_str = re.match(r"(?:>=)([0-9]+[\.0-9]*)", python_requires_str).groups()[0]
version = pkg_resources.parse_version(version_str)
return sdk_helpers.parse_version_object(version)
except AttributeError as e:
raise SDKException("'python_requires' version not given in correct format.")
def has_permissions(permissions, path):
"""
Raises an exception if the user does not have the given permissions to path
"""
LOG.debug("checking if: %s has correct permissions", path)
if not os.access(path, permissions):
if permissions is os.R_OK:
permissions = "READ"
elif permissions is os.W_OK:
permissions = "WRITE"
raise SDKException(
"User does not have {0} permissions for: {1}".format(
permissions, path))
def execute_command(self, args):
LOG.debug("called: CmdCodegen.execute_command()")
if args.reload:
if not args.package:
raise SDKException("'-p' must be specified when using '--reload'")
SDKException.command_ran = "{0} {1}".format(self.CMD_NAME, "--reload")
self._reload_package(args)
elif args.package:
SDKException.command_ran = "{0} {1}".format(self.CMD_NAME, "--package | -p")
self._gen_package(args)
elif not args.package and args.function:
SDKException.command_ran = "{0} {1}".format(self.CMD_NAME, "--function | -f")
self._gen_function(args)
else:
self.parser.print_help()
def perform_duplication_check(obj_type, obj_identifier, obj_type_name,
new_object_api_name, export):
"""Attempt to get the referenced object from the org_export
If the object is not found, return True.
If the object is found, raise an SDK Exception specifying the provided object name is not unique
and already exists on the system.
:param obj_type: The type name in the org export to search
:type obj_type: str
:param obj_identifier: The identifier for the given object
:type obj_identifier: str
:param obj_type_name: [description]
:type obj_type_name: str
:param new_object_api_name: [description]
:type new_object_api_name: str
:param export: The org export to search through
:type export: dict
:raises SDKException: If the provided object name is found then this function raises a SDK exception specifying this must be unique.
"""
# Perform a duplication check with the provided new name
try:
# Try to get a res obj with the new name
get_res_obj(obj_type,
obj_identifier,
obj_type_name, [new_object_api_name],
export,
include_api_name=False)
except SDKException:
# get_res_obj raises an exception if the object is not found
# normally this is good but for this unique use case
# we expect that the object will not be found and so catch and release the raised SDKException
return True
else:
# if get_res_obj does not raise an exception it means an object with that identifier exists
# and in this case we raise an SDKException as the new name provided for cloning needs to be unique
raise SDKException(
"The new name for a cloned object needs to be unique and a {} with the api name '{}' already exists"
.format(obj_type_name, new_object_api_name))
def test_fail_package_files_validate_customize_py(
fx_copy_fn_main_mock_integration):
filename = "customize.py"
attr_dict = sdk_validate_configs.package_files.get(filename)
path_file = os.path.join(
fx_copy_fn_main_mock_integration[1],
attr_dict.get("path").format(fx_copy_fn_main_mock_integration[0]))
# mock import definition parsing - mock raising an exception
with patch(
"resilient_sdk.util.sdk_validate_helpers.package_helpers.get_import_definition_from_customize_py"
) as mock_import_def:
mock_import_def.side_effect = SDKException("failed")
result = sdk_validate_helpers.package_files_validate_customize_py(
path_file, attr_dict)
assert len(result) == 1
result = result[0]
assert isinstance(result, SDKValidateIssue)
assert result.severity == SDKValidateIssue.SEVERITY_LEVEL_CRITICAL
def test_package_files_validate_improper_icon(
fx_copy_fn_main_mock_integration):
filename = "app_logo.png"
attr_dict = sdk_validate_configs.package_files.get(filename)
path_file = os.path.join(fx_copy_fn_main_mock_integration[1],
attr_dict.get("path"))
# mock import definition parsing - mock raising an exception
with patch(
"resilient_sdk.util.sdk_validate_helpers.package_helpers.get_icon"
) as mock_icon:
mock_icon.side_effect = SDKException("Failed for some reason")
result = sdk_validate_helpers.package_files_validate_icon(
path_file, attr_dict, filename)
assert len(result) == 1
result = result[0]
assert isinstance(result, SDKValidateIssue)
assert result.severity == SDKValidateIssue.SEVERITY_LEVEL_CRITICAL
assert "ERROR: Failed for some reason" == result.description
def read_local_exportfile(path_local_exportfile):
"""
Read export from given path
Return res export as dict
"""
# Get absolute path
path_local_exportfile = os.path.abspath(path_local_exportfile)
# Validate we can read it
validate_file_paths(os.R_OK, path_local_exportfile)
# Read the export file content.
if is_zipfile(path_local_exportfile):
# File is a zip file get unzipped content.
export_content = read_zip_file(path_local_exportfile,
RES_EXPORT_SUFFIX)
else:
# File is a assumed to be a text file read the export file content.
export_content = ''.join(read_file(path_local_exportfile))
if not export_content:
raise SDKException("Failed to read {0}".format(path_local_exportfile))
return json.loads(export_content)
def _reload_package(args):
old_params, path_customize_py_bak = [], ""
# Get + validate package, customize.py and setup.py paths
path_package = os.path.abspath(args.package)
# Get basename of path_to_src (version information is stripped from the basename).
path_package_basename = re.split(VERSION_REGEX, os.path.basename(path_package), 1)[0]
sdk_helpers.validate_dir_paths(os.R_OK, path_package)
path_customize_py = os.path.join(path_package, path_package_basename, package_helpers.PATH_CUSTOMIZE_PY)
sdk_helpers.validate_file_paths(os.W_OK, path_customize_py)
path_setup_py_file = os.path.join(path_package, package_helpers.PATH_SETUP_PY)
sdk_helpers.validate_file_paths(os.R_OK, path_setup_py_file)
# Set package + output args correctly (this handles if user runs 'codegen --reload -p .')
args.package = os.path.basename(path_package)
args.output = os.path.dirname(path_package)
LOG.info("'codegen --reload' started for '%s'", args.package)
# Load the customize.py module
customize_py_module = package_helpers.load_customize_py_module(path_customize_py, warn=False)
try:
# Get the 'old_params' from customize.py
old_params = customize_py_module.codegen_reload_data()
except AttributeError:
raise SDKException(u"Corrupt customize.py. No reload method found in {0}".format(path_customize_py))
if not old_params:
raise SDKException(u"No reload params found in {0}".format(path_customize_py))
# Rename the old customize.py with .bak
path_customize_py_bak = sdk_helpers.rename_to_bak_file(path_customize_py)
# If local export file exists then save it to a .bak file.
# (Older packages may not have the /util/data/export.res file)
path_export_res = os.path.join(path_package, path_package_basename,
package_helpers.PATH_UTIL_DATA_DIR,
package_helpers.BASE_NAME_LOCAL_EXPORT_RES)
if os.path.isfile(path_export_res):
path_export_res_bak = sdk_helpers.rename_to_bak_file(path_export_res)
else:
path_export_res_bak = None
try:
# Map command line arg name to dict key returned by codegen_reload_data() in customize.py
mapping_tuples = [
("messagedestination", "message_destinations"),
("function", "functions"),
("workflow", "workflows"),
("rule", "actions"),
("field", "incident_fields"),
("artifacttype", "incident_artifact_types"),
("datatable", "datatables"),
("task", "automatic_tasks"),
("script", "scripts")
]
# Merge old_params with new params specified on command line
args = CmdCodegen.merge_codegen_params(old_params, args, mapping_tuples)
# Parse the setup.py file
setup_py_attributes = package_helpers.parse_setup_py(path_setup_py_file, package_helpers.SUPPORTED_SETUP_PY_ATTRIBUTE_NAMES)
LOG.debug("Regenerating codegen '%s' package now", args.package)
# Regenerate the package
CmdCodegen._gen_package(args, setup_py_attributes=setup_py_attributes)
LOG.info("\nNOTE: Ensure the MANIFEST.in file includes line:\nrecursive-include %s/util *\n", args.package)
LOG.info("'codegen --reload' complete for '%s'", args.package)
except Exception as err:
LOG.error(u"Error running resilient-sdk codegen --reload\n\nERROR:%s", err)
# This is required in finally block as user may kill using keyboard interrupt
finally:
# If an error occurred, customize.py does not exist, rename the backup file to original
if not os.path.isfile(path_customize_py):
LOG.info(u"An error occurred. Renaming customize.py.bak to customize.py")
sdk_helpers.rename_file(path_customize_py_bak, package_helpers.BASE_NAME_CUSTOMIZE_PY)
if not os.path.isfile(path_export_res) and path_export_res_bak:
LOG.info(u"An error occurred. Renaming export.res.bak to export.res")
sdk_helpers.rename_file(path_export_res_bak, package_helpers.BASE_NAME_LOCAL_EXPORT_RES)
def _gen_package(args, setup_py_attributes={}):
LOG.info("Generating codegen package...")
if not sdk_helpers.is_valid_package_name(args.package):
raise SDKException(u"'{0}' is not a valid package name".format(args.package))
# Strip off version information, if present in package base folder, to get the package name.
package_name = re.split(VERSION_REGEX, args.package, 1)[0]
# Get base version if we are running against a package base folder with version.
base_version = ''.join(re.split(package_name, args.package))
# Get output_base, use args.output if defined, else current directory
output_base = args.output if args.output else os.curdir
output_base = os.path.abspath(output_base)
# If --exportfile is specified, read org_export from that file
if args.exportfile:
LOG.info("Using local export file: %s", args.exportfile)
org_export = sdk_helpers.read_local_exportfile(args.exportfile)
else:
# Instantiate connection to the Resilient Appliance
res_client = sdk_helpers.get_resilient_client()
# Generate + get latest export from Resilient Server
org_export = sdk_helpers.get_latest_org_export(res_client)
# Get data required for Jinja2 templates from export
jinja_data = sdk_helpers.get_from_export(org_export,
message_destinations=args.messagedestination,
functions=args.function,
workflows=args.workflow,
rules=args.rule,
fields=args.field,
artifact_types=args.artifacttype,
datatables=args.datatable,
tasks=args.task,
scripts=args.script)
# Get 'minified' version of the export. This is used in customize.py
jinja_data["export_data"] = sdk_helpers.minify_export(org_export,
message_destinations=sdk_helpers.get_object_api_names(ResilientObjMap.MESSAGE_DESTINATIONS, jinja_data.get("message_destinations")),
functions=sdk_helpers.get_object_api_names(ResilientObjMap.FUNCTIONS, jinja_data.get("functions")),
workflows=sdk_helpers.get_object_api_names(ResilientObjMap.WORKFLOWS, jinja_data.get("workflows")),
rules=sdk_helpers.get_object_api_names(ResilientObjMap.RULES, jinja_data.get("rules")),
fields=jinja_data.get("all_fields"),
artifact_types=sdk_helpers.get_object_api_names(ResilientObjMap.INCIDENT_ARTIFACT_TYPES, jinja_data.get("artifact_types")),
datatables=sdk_helpers.get_object_api_names(ResilientObjMap.DATATABLES, jinja_data.get("datatables")),
tasks=sdk_helpers.get_object_api_names(ResilientObjMap.TASKS, jinja_data.get("tasks")),
phases=sdk_helpers.get_object_api_names(ResilientObjMap.PHASES, jinja_data.get("phases")),
scripts=sdk_helpers.get_object_api_names(ResilientObjMap.SCRIPTS, jinja_data.get("scripts")))
# Add package_name to jinja_data
jinja_data["package_name"] = package_name
# Add version
jinja_data["version"] = setup_py_attributes.get("version", package_helpers.MIN_SETUP_PY_VERSION)
# Validate we have write permissions
sdk_helpers.validate_dir_paths(os.W_OK, output_base)
# Join package_name to output base (add base version if running against a folder which includes a version).
output_base = os.path.join(output_base, package_name+base_version)
# If the output_base directory does not exist, create it
if not os.path.exists(output_base):
os.makedirs(output_base)
# Instansiate Jinja2 Environment with path to Jinja2 templates
jinja_env = sdk_helpers.setup_jinja_env("data/codegen/templates/package_template")
# This dict maps our package file structure to Jinja2 templates
package_mapping_dict = {
"MANIFEST.in": ("MANIFEST.in.jinja2", jinja_data),
"README.md": ("README.md.jinja2", jinja_data),
"setup.py": ("setup.py.jinja2", jinja_data),
"tox.ini": ("tox.ini.jinja2", jinja_data),
"Dockerfile": ("Dockerfile.jinja2", jinja_data),
"entrypoint.sh": ("entrypoint.sh.jinja2", jinja_data),
"apikey_permissions.txt": ("apikey_permissions.txt.jinja2", jinja_data),
"data": {},
"icons": {
"company_logo.png": package_helpers.PATH_DEFAULT_ICON_COMPANY_LOGO,
"app_logo.png": package_helpers.PATH_DEFAULT_ICON_EXTENSION_LOGO,
},
"doc": {
"screenshots": {
"main.png": package_helpers.PATH_DEFAULT_SCREENSHOT
}
},
package_name: {
"__init__.py": ("package/__init__.py.jinja2", jinja_data),
"LICENSE": ("package/LICENSE.jinja2", jinja_data),
"components": {
"__init__.py": ("package/components/__init__.py.jinja2", jinja_data),
},
"util": {
"data": {
"export.res": ("package/util/data/export.res.jinja2", jinja_data)
},
"__init__.py": ("package/util/__init__.py.jinja2", jinja_data),
"config.py": ("package/util/config.py.jinja2", jinja_data),
"customize.py": ("package/util/customize.py.jinja2", jinja_data),
"selftest.py": ("package/util/selftest.py.jinja2", jinja_data),
}
}
}
# If there are Functions, add a 'tests' directory
if jinja_data.get("functions"):
package_mapping_dict["tests"] = {}
# Loop each Function
for f in jinja_data.get("functions"):
# Add package_name to function data
f["package_name"] = package_name
# Generate function_component.py file name
file_name = u"funct_{0}.py".format(f.get("export_key"))
# Add to 'components' directory
package_mapping_dict[package_name]["components"][file_name] = ("package/components/function.py.jinja2", f)
# Add to 'tests' directory
package_mapping_dict["tests"][u"test_{0}".format(file_name)] = ("tests/test_function.py.jinja2", f)
for w in jinja_data.get("workflows"):
# Generate wf_xx.md file name
file_name = u"wf_{0}.md".format(w.get(ResilientObjMap.WORKFLOWS))
# Add workflow to data directory
package_mapping_dict["data"][file_name] = ("data/workflow.md.jinja2", w)
newly_generated_files, skipped_files = CmdCodegen.render_jinja_mapping(
jinja_mapping_dict=package_mapping_dict,
jinja_env=jinja_env,
target_dir=output_base,
package_dir=output_base)
# Log new and skipped files
if newly_generated_files:
LOG.debug("Newly generated files:\n\t> %s", "\n\t> ".join(newly_generated_files))
if skipped_files:
LOG.debug("Files Skipped:\n\t> %s", "\n\t> ".join(skipped_files))
LOG.info("'codegen' complete for '%s'", package_name)
def _set_version(args):
new_version = args.set_version
if not sdk_helpers.is_valid_version_syntax(new_version):
raise SDKException(
"{0} is not a valid version".format(new_version))
new_version_int = list(map(int, (re.findall(r"\d+", new_version))))
# Get absolute path_to_src
path_to_src = os.path.abspath(args.package)
# Get path to setup.py file
path_setup_py_file = os.path.join(path_to_src,
package_helpers.BASE_NAME_SETUP_PY)
# Parse the setup.py file
setup_py_attributes = package_helpers.parse_setup_py(
path_setup_py_file,
package_helpers.SUPPORTED_SETUP_PY_ATTRIBUTE_NAMES)
package_name = setup_py_attributes.get("name", "")
LOG.info("Setting Resilient Platform version for %s to %s",
package_name, new_version)
# Get the customize file location.
path_customize_py = package_helpers.get_configuration_py_file_path(
"customize", setup_py_attributes)
# Get customize.py ImportDefinition
customize_py_import_definition = package_helpers.get_import_definition_from_customize_py(
path_customize_py)
old_version = customize_py_import_definition["server_version"][
"version"]
LOG.info("Old Version: %s", old_version)
LOG.info("New Version: %s", new_version)
# Set the new version
customize_py_import_definition["server_version"][
"version"] = new_version
customize_py_import_definition["server_version"][
"major"] = new_version_int[0]
customize_py_import_definition["server_version"][
"minor"] = new_version_int[1]
customize_py_import_definition["server_version"][
"build_number"] = new_version_int[2]
LOG.info("Loading old customize.py file")
# Load the customize.py module
customize_py_module = package_helpers.load_customize_py_module(
path_customize_py, warn=False)
# Get the 'old_params' from customize.py
old_params = customize_py_module.codegen_reload_data()
# Rename the old customize.py with .bak
path_customize_py_bak = sdk_helpers.rename_to_bak_file(
path_customize_py)
# If local export file exists then save it to a .bak file.
# (Older packages may not have the /util/data/export.res file)
# Figure out the path of the files first
dir_customize_py = os.path.dirname(path_customize_py)
path_local_export_res = os.path.join(
dir_customize_py, package_helpers.PATH_LOCAL_EXPORT_RES)
path_local_export_res_bak = None
if os.path.isfile(path_local_export_res):
path_local_export_res_bak = sdk_helpers.rename_to_bak_file(
path_local_export_res)
try:
jinja_data = sdk_helpers.get_from_export(
customize_py_import_definition,
message_destinations=old_params.get("message_destinations"),
functions=old_params.get("functions"),
workflows=old_params.get("workflows"),
rules=old_params.get("actions"),
fields=old_params.get("incident_fields"),
artifact_types=old_params.get("incident_artifact_types"),
datatables=old_params.get("datatables"),
tasks=old_params.get("automatic_tasks"),
scripts=old_params.get("scripts"))
jinja_data["export_data"] = sdk_helpers.minify_export(
customize_py_import_definition,
message_destinations=sdk_helpers.get_object_api_names(
ResilientObjMap.MESSAGE_DESTINATIONS,
jinja_data.get("message_destinations")),
functions=sdk_helpers.get_object_api_names(
ResilientObjMap.FUNCTIONS, jinja_data.get("functions")),
workflows=sdk_helpers.get_object_api_names(
ResilientObjMap.WORKFLOWS, jinja_data.get("workflows")),
rules=sdk_helpers.get_object_api_names(
ResilientObjMap.RULES, jinja_data.get("rules")),
fields=jinja_data.get("all_fields"),
artifact_types=sdk_helpers.get_object_api_names(
ResilientObjMap.INCIDENT_ARTIFACT_TYPES,
jinja_data.get("artifact_types")),
datatables=sdk_helpers.get_object_api_names(
ResilientObjMap.DATATABLES, jinja_data.get("datatables")),
tasks=sdk_helpers.get_object_api_names(
ResilientObjMap.TASKS, jinja_data.get("tasks")),
phases=sdk_helpers.get_object_api_names(
ResilientObjMap.PHASES, jinja_data.get("phases")),
scripts=sdk_helpers.get_object_api_names(
ResilientObjMap.SCRIPTS, jinja_data.get("scripts")))
# Add package_name to jinja_data
jinja_data["package_name"] = package_name
# Add version
jinja_data["version"] = setup_py_attributes.get(
"version", package_helpers.MIN_SETUP_PY_VERSION)
# Instansiate Jinja2 Environment with path to Jinja2 templates for customize.py
jinja_env = sdk_helpers.setup_jinja_env(
"data/codegen/templates/package_template/package/util")
jinja_template = jinja_env.get_template("customize.py.jinja2")
LOG.info("Writing new customize.py file")
# Render & write jinja2 template
jinja_rendered_text = jinja_template.render(jinja_data)
sdk_helpers.write_file(path_customize_py, jinja_rendered_text)
# Instansiate Jinja2 Environment with path to Jinja2 templates for /util/dat/export.res
#jinja_env = sdk_helpers.setup_jinja_env("data/codegen/templates/package_template/package/util/data")
jinja_template = jinja_env.get_template("/data/export.res.jinja2")
LOG.debug("Writing new /util/data/export.res file")
# Render jinja2 template
jinja_rendered_text = jinja_template.render(jinja_data)
# Make sure the /util/data directory is there if it is not
dir_local_export_res = os.path.dirname(path_local_export_res)
if not os.path.exists(dir_local_export_res):
os.makedirs(dir_local_export_res)
# Write the file
sdk_helpers.write_file(path_local_export_res, jinja_rendered_text)
LOG.info("'dev --set-version' complete for '%s'", package_name)
except Exception as err:
LOG.error(
u"Error running resilient-sdk dev --set-version\n\nERROR:%s",
err)
# This is required in finally block as user may kill using keyboard interrupt
finally:
# If an error occurred, customize.py does not exist, rename the backup file to original
if not os.path.isfile(path_customize_py):
LOG.info(
u"An error occurred. Renaming customize.py.bak to customize.py"
)
sdk_helpers.rename_file(path_customize_py_bak,
package_helpers.BASE_NAME_CUSTOMIZE_PY)
if path_local_export_res_bak and not os.path.isfile(
path_local_export_res):
LOG.info(
u"An error occurred. Renaming /util/data/export.res.bak to export.res"
)
sdk_helpers.rename_file(
path_local_export_res_bak,
package_helpers.BASE_NAME_LOCAL_EXPORT_RES)
def execute_command(self, args):
LOG.debug("called: ConnectorCodegenCmd.execute_command()")
LOG.info(args)
if args.package:
SDKException.command_ran = "{0} {1}".format(
self.CMD_NAME, "--package | -p")
LOG.info("%s %s" %
("Generating a package of type", DEFAULT_CONNECTOR))
###
# Perform checks
###
if os.path.exists(args.package):
raise SDKException(
u"'{0}' already exists. Add --reload flag to regenerate it"
.format(args.package))
if not sdk_helpers.is_valid_package_name(args.package):
raise SDKException(u"'{0}' is not a valid package name".format(
args.package))
###
# Gather data needed for generation
###
# The package_name will be specified in the args
package_name = args.package
# Get output_base, use args.output if defined, else current directory
output_base = args.output if args.output else os.curdir
output_base = os.path.join(os.path.abspath(output_base),
package_name)
jinja_data = {"package_name": package_name}
# If the output_base directory does not exist, create it
if not os.path.exists(output_base):
os.makedirs(output_base)
from jinja2 import Environment, PackageLoader
jinja_env = Environment(
# Loads Jinja Templates in cp4s_connector_sdk/<<relative_path_to_templates>>
loader=PackageLoader(
"cp4s_connector_sdk",
get_jinja_env_location(connecter_type=args.connectortype)),
trim_blocks=True, # First newline after a block is removed
# Leading spaces and tabs are stripped from the start of a line to a block
lstrip_blocks=True,
keep_trailing_newline=
True # Preserve the trailing newline when rendering templates
)
# Add custom filters to our jinja_env
sdk_helpers.add_filters_to_jinja_env(jinja_env)
# Assign one of the mapping dict function to connector_mapping_collector so that below we call one thing
# but here we define which thing will be called
# TODO: Review
connector_mapping_collector = get_car_connector_mapping_dict
# Prepare a mapping dict based on type of connector
# This dict maps our package file structure to Jinja2 templates
# TODO: Refactor as there can be a UDI or CAR package_mapping_dict
package_mapping_dict = connector_mapping_collector(jinja_data)
###
# Generate a connector package using jinja
###
newly_generated_files, skipped_files = CmdCodegen.render_jinja_mapping(
jinja_mapping_dict=package_mapping_dict,
jinja_env=jinja_env,
target_dir=output_base,
package_dir=output_base)
###
# Report the files that were and were not built
###
if newly_generated_files:
LOG.debug("Newly generated files:\n\t> %s",
"\n\t> ".join(newly_generated_files))
if skipped_files:
LOG.debug("Files Skipped:\n\t> %s",
"\n\t> ".join(skipped_files))
LOG.info("%s %s" % ("Codegen run finished for ", package_name))
else:
self.parser.print_help()
def _get_results_from_log_file(cls, args):
"""
- Gets all function names from the payload_samples directory
- Traverses the file at the path specified by args.gather_results (in a reversed order)
- Looks for lines containing ``[<fn_name>] Result: {'version': 2.0, 'success': True...``
- Parses it and generates an output_json_example.json and output_json_schema.json file for each ``Result`` found
- Uses the libary ``genson`` to generate the JSON schema from a Python Dictionary
:param args: (required) the cmd line arguments
:type args: argparse.ArgumentParser
:raises: an SDKException if args.package is not a valid path
"""
# Check if Python >= MIN_SUPPORTED_PY_VERSION
if not sdk_helpers.is_python_min_supported_version(constants.ERROR_WRONG_PYTHON_VERSION):
raise SDKException(constants.ERROR_WRONG_PYTHON_VERSION)
path_package = os.path.abspath(args.package)
path_log_file = args.gather_results
path_payload_samples_dir = os.path.join(path_package, package_helpers.BASE_NAME_PAYLOAD_SAMPLES_DIR)
LOG.debug("\nPath to project: %s", path_package)
sdk_helpers.validate_dir_paths(os.W_OK, path_package)
package_name = package_helpers.get_package_name(path_package)
LOG.info("'codegen %s' started for '%s'", constants.SUB_CMD_OPT_GATHER_RESULTS, package_name)
try:
sdk_helpers.validate_dir_paths(os.W_OK, path_payload_samples_dir)
except SDKException as e:
if constants.ERROR_NOT_FIND_DIR in e.message:
LOG.warning("WARNING: no '%s' found. Running 'codegen --reload' to create the default missing files\n%s", package_helpers.BASE_NAME_PAYLOAD_SAMPLES_DIR, constants.LOG_DIVIDER)
args.reload = True
cls._reload_package(args)
LOG.warning(constants.LOG_DIVIDER)
else:
raise e
functions_that_need_payload_samples = args.function if args.function else os.listdir(path_payload_samples_dir)
results_scraped = sdk_helpers.scrape_results_from_log_file(path_log_file)
for fn_name in functions_that_need_payload_samples:
fn_results = results_scraped.get(fn_name)
if not fn_results:
package_helpers.color_output("WARNING: No results could be found for '{0}' in '{1}'".format(fn_name, path_log_file), constants.VALIDATE_LOG_LEVEL_WARNING, do_print=True)
continue
LOG.info("Results found for '[%s]'", fn_name)
path_payload_samples_fn_name = os.path.join(path_payload_samples_dir, fn_name)
path_output_json_example = os.path.join(path_payload_samples_fn_name, package_helpers.BASE_NAME_PAYLOAD_SAMPLES_EXAMPLE)
path_output_json_schema = os.path.join(path_payload_samples_fn_name, package_helpers.BASE_NAME_PAYLOAD_SAMPLES_SCHEMA)
path_output_json_example_bak = sdk_helpers.rename_to_bak_file(path_output_json_example)
path_output_json_schema_bak = sdk_helpers.rename_to_bak_file(path_output_json_schema)
try:
LOG.debug("Writing JSON example file for '%s' to '%s'", fn_name, path_output_json_example)
sdk_helpers.write_file(path_output_json_example, json.dumps(fn_results, indent=2))
LOG.debug("Writing JSON schema file for '%s' to '%s'", fn_name, path_output_json_schema)
builder = CustomSchemaBuilder(schema_uri=constants.CODEGEN_JSON_SCHEMA_URI)
main_genson_builder_overwrites(builder)
builder.add_object(fn_results)
sdk_helpers.write_file(path_output_json_schema, builder.to_json(indent=2))
finally:
if not os.path.isfile(path_output_json_example) and path_output_json_example_bak:
LOG.info(u"An error occurred. Renaming %s.bak to %s", package_helpers.BASE_NAME_PAYLOAD_SAMPLES_EXAMPLE, package_helpers.BASE_NAME_PAYLOAD_SAMPLES_EXAMPLE)
sdk_helpers.rename_file(path_output_json_example_bak, package_helpers.BASE_NAME_PAYLOAD_SAMPLES_EXAMPLE)
if not os.path.isfile(path_output_json_schema) and path_output_json_schema_bak:
LOG.info(u"An error occurred. Renaming %s.bak to %s", package_helpers.BASE_NAME_PAYLOAD_SAMPLES_SCHEMA, package_helpers.BASE_NAME_PAYLOAD_SAMPLES_SCHEMA)
sdk_helpers.rename_file(path_output_json_schema_bak, package_helpers.BASE_NAME_PAYLOAD_SAMPLES_SCHEMA)
LOG.info("'codegen %s' complete for '%s'", constants.SUB_CMD_OPT_GATHER_RESULTS, package_name)
def _reload_package(args):
old_params, path_customize_py_bak = [], ""
# Get absolute path to package
path_package = os.path.abspath(args.package)
LOG.debug("\nPath to project: %s", path_package)
# Ensure the package directory exists and we have WRITE access
sdk_helpers.validate_dir_paths(os.W_OK, path_package)
path_setup_py_file = os.path.join(path_package, package_helpers.BASE_NAME_SETUP_PY)
package_name = package_helpers.get_package_name(path_package)
if not sdk_helpers.is_valid_package_name(package_name):
raise SDKException(u"'{0}' is not a valid package name. 'name' attribute in setup.py file is not valid or not specified".format(package_name))
LOG.debug("\nProject name: %s", package_name)
# Generate path to customize.py file + validate we have permissions to read it
path_customize_py = os.path.join(path_package, package_name, package_helpers.PATH_CUSTOMIZE_PY)
sdk_helpers.validate_file_paths(os.W_OK, path_customize_py)
# Set package + output args correctly (this handles if user runs 'codegen --reload -p .')
args.package = package_name
args.output = path_package
LOG.info("'codegen --reload' started for '%s'", args.package)
# Load the customize.py module
customize_py_module = package_helpers.load_customize_py_module(path_customize_py, warn=False)
try:
# Get the 'old_params' from customize.py
old_params = customize_py_module.codegen_reload_data()
except AttributeError:
raise SDKException(u"Corrupt customize.py. No reload method found in {0}".format(path_customize_py))
if not old_params:
raise SDKException(u"No reload params found in {0}".format(path_customize_py))
# Rename the old customize.py with .bak
path_customize_py_bak = sdk_helpers.rename_to_bak_file(path_customize_py)
# If local export file exists then save it to a .bak file.
# (Older packages may not have the /util/data/export.res file)
path_export_res = os.path.join(path_package, package_name,
package_helpers.PATH_UTIL_DATA_DIR,
package_helpers.BASE_NAME_LOCAL_EXPORT_RES)
if os.path.isfile(path_export_res):
path_export_res_bak = sdk_helpers.rename_to_bak_file(path_export_res)
else:
path_export_res_bak = None
try:
# Map command line arg name to dict key returned by codegen_reload_data() in customize.py
mapping_tuples = [
("messagedestination", "message_destinations"),
("function", "functions"),
("workflow", "workflows"),
("rule", "actions"),
("field", "incident_fields"),
("artifacttype", "incident_artifact_types"),
("incidenttype", "incident_types"),
("datatable", "datatables"),
("task", "automatic_tasks"),
("script", "scripts"),
("playbook", "playbooks")
]
# Merge old_params with new params specified on command line
args = CmdCodegen.merge_codegen_params(old_params, args, mapping_tuples)
# Parse the setup.py file
setup_py_attributes = package_helpers.parse_setup_py(path_setup_py_file, package_helpers.SUPPORTED_SETUP_PY_ATTRIBUTE_NAMES)
LOG.debug("Regenerating codegen '%s' package now", args.package)
# Regenerate the package
path_reloaded = CmdCodegen._gen_package(args, setup_py_attributes=setup_py_attributes)
LOG.info("\nNOTE: Ensure the MANIFEST.in file includes line:\nrecursive-include %s/util *\n", args.package)
LOG.info("'codegen --reload' complete for '%s'", args.package)
return path_reloaded
# This is required in finally block as user may kill using keyboard interrupt
finally:
# If an error occurred, customize.py does not exist, rename the backup file to original
if not os.path.isfile(path_customize_py):
LOG.info(u"An error occurred. Renaming customize.py.bak to customize.py")
sdk_helpers.rename_file(path_customize_py_bak, package_helpers.BASE_NAME_CUSTOMIZE_PY)
if not os.path.isfile(path_export_res) and path_export_res_bak:
LOG.info(u"An error occurred. Renaming export.res.bak to export.res")
sdk_helpers.rename_file(path_export_res_bak, package_helpers.BASE_NAME_LOCAL_EXPORT_RES)
def _gen_package(args, setup_py_attributes={}):
LOG.info("Generating codegen package...")
sdk_helpers.is_python_min_supported_version()
if os.path.exists(args.package) and not args.reload:
raise SDKException(u"'{0}' already exists. Add --reload flag to regenerate it".format(args.package))
if not sdk_helpers.is_valid_package_name(args.package):
raise SDKException(u"'{0}' is not a valid package name".format(args.package))
# The package_name will be specified in the args
package_name = args.package
# Get output_base, use args.output if defined, else current directory
output_base = args.output if args.output else os.curdir
output_base = os.path.abspath(output_base)
# If --exportfile is specified, read org_export from that file
if args.exportfile:
LOG.info("Using local export file: %s", args.exportfile)
org_export = sdk_helpers.read_local_exportfile(args.exportfile)
else:
# Instantiate connection to the Resilient Appliance
res_client = sdk_helpers.get_resilient_client(path_config_file=args.config)
# Generate + get latest export from Resilient Server
org_export = sdk_helpers.get_latest_org_export(res_client)
# Get data required for Jinja2 templates from export
jinja_data = sdk_helpers.get_from_export(org_export,
message_destinations=args.messagedestination,
functions=args.function,
workflows=args.workflow,
rules=args.rule,
fields=args.field,
artifact_types=args.artifacttype,
datatables=args.datatable,
tasks=args.task,
scripts=args.script,
incident_types=args.incidenttype,
playbooks=args.playbook)
# Get 'minified' version of the export. This is used in customize.py
jinja_data["export_data"] = sdk_helpers.minify_export(org_export,
message_destinations=sdk_helpers.get_object_api_names(ResilientObjMap.MESSAGE_DESTINATIONS, jinja_data.get("message_destinations")),
functions=sdk_helpers.get_object_api_names(ResilientObjMap.FUNCTIONS, jinja_data.get("functions")),
workflows=sdk_helpers.get_object_api_names(ResilientObjMap.WORKFLOWS, jinja_data.get("workflows")),
rules=sdk_helpers.get_object_api_names(ResilientObjMap.RULES, jinja_data.get("rules")),
fields=jinja_data.get("all_fields"),
artifact_types=sdk_helpers.get_object_api_names(ResilientObjMap.INCIDENT_ARTIFACT_TYPES, jinja_data.get("artifact_types")),
datatables=sdk_helpers.get_object_api_names(ResilientObjMap.DATATABLES, jinja_data.get("datatables")),
tasks=sdk_helpers.get_object_api_names(ResilientObjMap.TASKS, jinja_data.get("tasks")),
phases=sdk_helpers.get_object_api_names(ResilientObjMap.PHASES, jinja_data.get("phases")),
scripts=sdk_helpers.get_object_api_names(ResilientObjMap.SCRIPTS, jinja_data.get("scripts")),
incident_types=sdk_helpers.get_object_api_names(ResilientObjMap.INCIDENT_TYPES, jinja_data.get("incident_types")),
playbooks=sdk_helpers.get_object_api_names(ResilientObjMap.PLAYBOOKS, jinja_data.get("playbooks")))
# Add package_name to jinja_data
jinja_data["package_name"] = package_name
# Add version
jinja_data["version"] = setup_py_attributes.get("version", package_helpers.MIN_SETUP_PY_VERSION)
jinja_data["resilient_libraries_version"] = sdk_helpers.get_resilient_libraries_version_to_use()
# Validate we have write permissions
sdk_helpers.validate_dir_paths(os.W_OK, output_base)
if not args.reload:
# If this is not a reload, join package_name to output base
output_base = os.path.join(output_base, package_name)
# If the output_base directory does not exist, create it
if not os.path.exists(output_base):
os.makedirs(output_base)
# Instansiate Jinja2 Environment with path to Jinja2 templates
jinja_env = sdk_helpers.setup_jinja_env(constants.PACKAGE_TEMPLATE_PATH)
# This dict maps our package file structure to Jinja2 templates
package_mapping_dict = {
"MANIFEST.in": ("MANIFEST.in.jinja2", jinja_data),
"README.md": ("README.md.jinja2", jinja_data),
"setup.py": ("setup.py.jinja2", jinja_data),
"tox.ini": ("tox.ini.jinja2", jinja_data),
"Dockerfile": ("Dockerfile.jinja2", jinja_data),
"entrypoint.sh": ("entrypoint.sh.jinja2", jinja_data),
"apikey_permissions.txt": ("apikey_permissions.txt.jinja2", jinja_data),
"data": {},
"icons": {
"company_logo.png": package_helpers.PATH_DEFAULT_ICON_COMPANY_LOGO,
"app_logo.png": package_helpers.PATH_DEFAULT_ICON_EXTENSION_LOGO,
},
"doc": {
"screenshots": {
"main.png": package_helpers.PATH_DEFAULT_SCREENSHOT
}
},
package_name: {
"__init__.py": ("package/__init__.py.jinja2", jinja_data),
"LICENSE": ("package/LICENSE.jinja2", jinja_data),
"components": {
"__init__.py": ("package/components/__init__.py.jinja2", jinja_data),
},
"util": {
"data": {
"export.res": ("package/util/data/export.res.jinja2", jinja_data)
},
"__init__.py": ("package/util/__init__.py.jinja2", jinja_data),
"config.py": ("package/util/config.py.jinja2", jinja_data),
"customize.py": ("package/util/customize.py.jinja2", jinja_data),
"selftest.py": ("package/util/selftest.py.jinja2", jinja_data),
}
}
}
# If there are Functions, add a 'tests' and a 'payload_samples' directory (if in dev mode)
if jinja_data.get("functions"):
package_mapping_dict["tests"] = {}
package_mapping_dict[package_helpers.BASE_NAME_PAYLOAD_SAMPLES_DIR] = {}
# Get a list of function names in export.
fn_names = [f.get(ResilientObjMap.FUNCTIONS) for f in jinja_data.get("functions")]
# Loop each Function
for f in jinja_data.get("functions"):
# Add package_name to function data
f["package_name"] = package_name
# Get function name
fn_name = f.get(ResilientObjMap.FUNCTIONS)
# Generate funct_function_component.py file name
# Don't add prefix if function name already begins with "func_" or "funct_".
if re.search(r"^(func|funct)_", fn_name):
file_name = u"{0}.py".format(fn_name)
else:
file_name = u"funct_{0}.py".format(fn_name)
# Check if file_name without extension already exists in functions names list.
if os.path.splitext(file_name)[0] in fn_names:
raise SDKException(u"File name '{0}' already in use please rename the function '{1}'."
.format(file_name, fn_name))
# Add an 'atomic function' to 'components' directory else add a 'normal function'
package_mapping_dict[package_name]["components"][file_name] = ("package/components/atomic_function.py.jinja2", f)
# Add to 'tests' directory
package_mapping_dict["tests"][u"test_{0}".format(file_name)] = ("tests/test_function.py.jinja2", f)
# Add a 'payload_samples/fn_name' directory and the files to it
CmdCodegen.add_payload_samples(package_mapping_dict, fn_name, f)
# Get a list of workflow names in export.
wf_names = [w.get(ResilientObjMap.WORKFLOWS) for w in jinja_data.get("workflows")]
for w in jinja_data.get("workflows"):
# Get workflow name
wf_name = w.get(ResilientObjMap.WORKFLOWS)
# Generate wf_xx.md file name
# Don't add prefix if workflow name already begins with "wf_".
if re.search(r"^wf_", wf_name):
file_name = u"{0}.md".format(wf_name)
else:
file_name = u"wf_{0}.md".format(wf_name)
# Check if file_name without extension already exists in workflow names list.
if os.path.splitext(file_name)[0] in wf_names:
raise SDKException(u"File name '{0}' already in use please recreate the workflow '{1}'."
.format(file_name, wf_name))
# Add workflow to data directory
package_mapping_dict["data"][file_name] = ("data/workflow.md.jinja2", w)
newly_generated_files, skipped_files = CmdCodegen.render_jinja_mapping(
jinja_mapping_dict=package_mapping_dict,
jinja_env=jinja_env,
target_dir=output_base,
package_dir=output_base)
# Log new and skipped files
if newly_generated_files:
LOG.debug("Newly generated files:\n\t> %s", "\n\t> ".join(newly_generated_files))
if skipped_files:
LOG.debug("Files Skipped:\n\t> %s", "\n\t> ".join(skipped_files))
LOG.info("'codegen' complete for '%s'", package_name)
return output_base
def create_extension(path_setup_py_file,
path_apikey_permissions_file,
output_dir,
path_built_distribution=None,
path_extension_logo=None,
path_company_logo=None,
custom_display_name=None,
repository_name=None,
keep_build_dir=False):
"""
TODO: update this docstring to new standard format
Function that creates The App.zip file from the given setup.py, customize and config files
and copies it to the output_dir. Returns the path to the App.zip
- path_setup_py_file [String]: abs path to the setup.py file
- path_apikey_permissions_file [String]: abs path to the apikey_permissions.txt file
- output_dir [String]: abs path to the directory the App.zip should be produced
- path_built_distribution [String]: abs path to a tar.gz Built Distribution
- if provided uses that .tar.gz
- else looks for it in the output_dir. E.g: output_dir/package_name.tar.gz
- path_extension_logo [String]: abs path to the app_logo.png. Has to be 200x72 and a .png file
- if not provided uses default icon
- path_company_logo [String]: abs path to the company_logo.png. Has to be 100x100 and a .png file
- if not provided uses default icon
- custom_display_name [String]: will give the App that display name. Default: name from setup.py file
- repository_name [String]: will over-ride the container repository name for the App. Default: 'ibmresilient'
- keep_build_dir [Boolean]: if True, build/ will not be remove. Default: False
"""
LOG.info("Creating App")
# Variables to hold path of files for customize and config as defined in setup.py.
# Set initially default to 'None', actual paths will be calculated later.
path_customize_py_file = None
path_config_py_file = None
# Ensure the output_dir exists, we have WRITE access and ensure we can READ setup.py and apikey_permissions.txt
# files.
sdk_helpers.validate_dir_paths(os.W_OK, output_dir)
sdk_helpers.validate_file_paths(os.R_OK, path_setup_py_file,
path_apikey_permissions_file)
# Parse the setup.py file
setup_py_attributes = parse_setup_py(path_setup_py_file,
SUPPORTED_SETUP_PY_ATTRIBUTE_NAMES)
# Validate setup.py attributes
# Validate the name attribute. Raise exception if invalid
if not sdk_helpers.is_valid_package_name(setup_py_attributes.get("name")):
raise SDKException(
"'{0}' is not a valid App name. The name attribute must be defined and can only include 'a-z and _'.\nUpdate this value in the setup.py file located at: {1}"
.format(setup_py_attributes.get("name"), path_setup_py_file))
# Validate the version attribute. Raise exception if invalid
if not sdk_helpers.is_valid_version_syntax(
setup_py_attributes.get("version")):
raise SDKException(
"'{0}' is not a valid App version syntax. The version attribute must be defined. Example: version=\"1.0.0\".\nUpdate this value in the setup.py file located at: {1}"
.format(setup_py_attributes.get("version"), path_setup_py_file))
# Validate the url supplied in the setup.py file, set to an empty string if not valid
if not sdk_helpers.is_valid_url(setup_py_attributes.get("url")):
LOG.warning("WARNING: '%s' is not a valid url. Ignoring.",
setup_py_attributes.get("url"))
setup_py_attributes["url"] = ""
# Get the tag name
tag_name = setup_py_attributes.get("name")
# Get the customize file location.
path_customize_py_file = get_configuration_py_file_path(
"customize", setup_py_attributes)
# Get the config file location.
path_config_py_file = get_configuration_py_file_path(
"config", setup_py_attributes)
# Get ImportDefinition from the discovered customize file.
if path_customize_py_file:
import_definition = get_import_definition_from_customize_py(
path_customize_py_file)
else:
# No 'customize.py' file found generate import definition with just mimimum server version.
import_definition = {'server_version': IMPORT_MIN_SERVER_VERSION}
# Add the tag to the import defintion
import_definition = add_tag_to_import_definition(tag_name,
SUPPORTED_RES_OBJ_NAMES,
import_definition)
# Parse the app.configs from the discovered config file
if path_config_py_file:
app_configs = get_configs_from_config_py(path_config_py_file)
else:
# No config file file found generate an empty definition.
app_configs = ("", [])
# Parse the api key permissions from the apikey_permissions.txt file
apikey_permissions = get_apikey_permissions(path_apikey_permissions_file)
# Generate the name for the extension
extension_name = "{0}-{1}".format(setup_py_attributes.get("name"),
setup_py_attributes.get("version"))
# Generate the uuid
uuid = sdk_helpers.generate_uuid_from_string(
setup_py_attributes.get("name"))
# Set the container repository name to default if value not passed in as argument.
if not repository_name:
repository_name = REPOSITORY_NAME
# Generate paths to the directories and files we will use in the build directory
path_build = os.path.join(output_dir, BASE_NAME_BUILD)
path_extension_json = os.path.join(path_build, BASE_NAME_EXTENSION_JSON)
path_export_res = os.path.join(path_build, BASE_NAME_EXPORT_RES)
try:
# If there is an old build directory, remove it first
if os.path.exists(path_build):
shutil.rmtree(path_build)
# Create the directories for the path "/build/"
os.makedirs(path_build)
# If no path_built_distribution is given, use the default: "<output_dir>/<package-name>.tar.gz"
if not path_built_distribution:
path_built_distribution = os.path.join(
output_dir, "{0}.tar.gz".format(extension_name))
# Validate the built distribution exists and we have READ access
sdk_helpers.validate_file_paths(os.R_OK, path_built_distribution)
# Copy the built distribution to the build dir and enforce rename to .tar.gz
shutil.copy(
path_built_distribution,
os.path.join(path_build, "{0}.tar.gz".format(extension_name)))
# Get the extension_logo (icon) and company_logo (author.icon) as base64 encoded strings
extension_logo = get_icon(
icon_name=os.path.basename(PATH_DEFAULT_ICON_EXTENSION_LOGO),
path_to_icon=path_extension_logo,
width_accepted=200,
height_accepted=72,
default_path_to_icon=PATH_DEFAULT_ICON_EXTENSION_LOGO)
company_logo = get_icon(
icon_name=os.path.basename(PATH_DEFAULT_ICON_COMPANY_LOGO),
path_to_icon=path_company_logo,
width_accepted=100,
height_accepted=100,
default_path_to_icon=PATH_DEFAULT_ICON_COMPANY_LOGO)
# Get the display name
# Use --display-name if passed
# If not use 'display_name' attribute in setup.py
# If not set use the 'name' attribute in setup.py
display_name = custom_display_name or setup_py_attributes.get(
"display_name") or setup_py_attributes.get("name")
# Image string is all lowercase on quay.io
image_name = "{0}/{1}:{2}".format(repository_name,
setup_py_attributes.get("name"),
setup_py_attributes.get("version"))
image_name = image_name.lower()
# Generate the contents for the extension.json file
the_extension_json_file_contents = {
"author": {
"name": setup_py_attributes.get("author"),
"website": setup_py_attributes.get("url"),
"icon": {
"data": company_logo,
"media_type": "image/png"
}
},
"description": {
"content": setup_py_attributes.get("description"),
"format": "text"
},
"display_name": display_name,
"icon": {
"data": extension_logo,
"media_type": "image/png"
},
"long_description": {
"content":
"<div>{0}</div>".format(
setup_py_attributes.get("long_description")),
"format":
"html"
},
"minimum_resilient_version": {
"major":
import_definition.get("server_version").get("major", None),
"minor":
import_definition.get("server_version").get("minor", None),
"build_number":
import_definition.get("server_version").get(
"build_number", None),
"version":
import_definition.get("server_version").get("version", None)
},
"name": setup_py_attributes.get("name"),
"tag": {
"prefix": tag_name,
"name": tag_name,
"display_name": tag_name,
"uuid": uuid
},
"uuid": uuid,
"version": setup_py_attributes.get("version"),
"current_installation": {
"executables": [{
"name": setup_py_attributes.get("name"),
"image": image_name,
"config_string": app_configs[0],
"permission_handles": apikey_permissions,
"uuid": uuid
}]
}
}
# Write the executable.json file
sdk_helpers.write_file(
path_extension_json,
json.dumps(the_extension_json_file_contents, sort_keys=True))
# Write the customize ImportDefinition to the app*.zip export.res file
sdk_helpers.write_file(path_export_res,
json.dumps(import_definition, sort_keys=True))
# Copy the built distribution to the build dir, enforce rename to .tar.gz
shutil.copy(
path_built_distribution,
os.path.join(path_build, "{0}.tar.gz".format(extension_name)))
# create The Extension Zip by zipping the build directory
extension_zip_base_path = os.path.join(
output_dir, "{0}{1}".format(PREFIX_EXTENSION_ZIP, extension_name))
extension_zip_name = shutil.make_archive(
base_name=extension_zip_base_path,
format="zip",
root_dir=path_build)
path_the_extension_zip = os.path.join(extension_zip_base_path,
extension_zip_name)
except SDKException as err:
raise err
except Exception as err:
raise SDKException(err)
finally:
# Remove the build dir. Keep it if user passes --keep-build-dir
if not keep_build_dir:
shutil.rmtree(path_build)
LOG.info("App %s.zip created", "{0}{1}".format(PREFIX_EXTENSION_ZIP,
extension_name))
# Return the path to the extension zip
return path_the_extension_zip
def get_configs_from_config_py(path_config_py_file):
"""Returns a tuple (config_str, config_list). If no configs found, return ("", []).
Raises Exception if it fails to parse configs
- config_str: is the full string found in the config file
- apphost_config_str: is the full string found in the app host config file
- config_list: is a list of dict objects that contain each un-commented config
- Each dict object has the attributes: name, placeholder, env_name, section_name
"""
config_str, config_list = "", []
try:
# Get the module name from the config file path by getting basename and stripping '.py'
# from the rhs of the resulting string.
config_module = os.path.basename(path_config_py_file)[:-3]
# Import the config module
config_py = sdk_helpers.load_py_module(path_config_py_file,
config_module)
# Call config_section_data() to get the string containing the configs
config_str = config_py.config_section_data()
# Call apphost_config_section_data available to get app host config settings
try:
apphost_config_str = config_py.apphost_config_section_data()
except AttributeError:
# An app host config may not exist
apphost_config_str = None
# concatenate the config sections
clean_cfg_list = list(filter(None, [apphost_config_str, config_str]))
concat_cfg_str = '\n'.join(clean_cfg_list)
# Iterate over config sections and parse settings.
for cfg_str in clean_cfg_list:
# Instansiate a new configparser
config_parser = configparser.ConfigParser()
# Read and parse the configs from the config_str or apphost_config_str
if sys.version_info < (3, 2):
# config_parser.readfp() was deprecated and replaced with read_file in PY3.2
config_parser.readfp(StringIO(cfg_str))
else:
config_parser.read_file(StringIO(cfg_str))
# Get the configs from each section
for section_name in config_parser.sections():
parsed_configs = config_parser.items(section_name)
for config in parsed_configs:
config_list.append({
"name":
config[0],
"placeholder":
config[1],
"env_name":
"{0}_{1}".format(section_name.upper(),
config[0].upper()),
"section_name":
section_name
})
except ImportError as err:
raise SDKException(
u"Failed to load module '{0}' got error '{1}'".format(
config_module, err.__repr__()))
except Exception as err:
raise SDKException(
u"Failed to parse configs from the config file\nThe config file may be corrupt. Visit "
u"the App Exchange to contact the developer\nReason: {0}".format(
err))
return (concat_cfg_str, config_list)