def load_yamls(path):
yamls = [join(path, f) for f in listdir(path) if isfile(join(path, f)) if f.endswith('.yaml') or f.endswith('.yml')]
result = []
for yaml in yamls:
try:
result.append(read_yaml_file(yaml))
except ScannerError:
raise ScannerError('yaml is bad! %s' % yaml)
return result, yamls
def main(args):
logging.basicConfig(level=getattr(logging, args.log))
utils.GCLOUD_OPTIONS = utils.GcloudOptions(dry_run=args.dry_run,
gcloud_bin=args.gcloud_bin)
# Read and parse the project configuration YAML file.
all_projects = utils.resolve_env_vars(
utils.read_yaml_file(args.project_yaml))
if not all_projects:
logging.error('Error loading project YAML.')
return
logging.info('Validating project YAML against schema.')
try:
utils.validate_config_yaml(all_projects)
except jsonschema.exceptions.ValidationError as e:
logging.error('Error in YAML config: %s', e)
return
overall = all_projects['overall']
audit_logs_project = all_projects.get('audit_logs_project')
projects = []
# Always deploy the remote audit logs project first (if present).
if audit_logs_project:
projects.append(
ProjectConfig(overall=overall,
project=audit_logs_project,
audit_logs_project=None))
for project_config in all_projects.get('projects', []):
projects.append(
ProjectConfig(overall=overall,
project=project_config,
audit_logs_project=audit_logs_project))
# If resuming setup from a particular project, skip to that project.
if args.resume_from_project:
while (projects and
projects[0].project['project_id'] != args.resume_from_project):
skipped = projects.pop(0)
logging.info('Skipping project %s', skipped.project['project_id'])
if not projects:
logging.error('Project not found: %s', args.resume_from_project)
if projects:
starting_step = max(1, args.resume_from_step)
for config in projects:
logging.info('Setting up project %s', config.project['project_id'])
if not setup_new_project(config, starting_step):
# Don't attempt to deploy additional projects if one project failed.
return
starting_step = 1
else:
logging.error('No projects to deploy.')
def yaml2markdown_jinja(file, type):
# there is a problem with working directory, for now this script must be run from scripts directory
env = Environment(loader=FileSystemLoader('templates'))
fields = read_yaml_file(file)
if type == "detectionrule" or type == "DR":
alert = fields
template = env.get_template('markdown_alert_template.md.j2')
parent_title = "Detection_Rules"
sigma_rule = read_rule_file(file)
alert.update({'sigma_rule': sigma_rule})
outputs = ["es-qs", "xpack-watcher", "graylog"]
for output in outputs:
cmd = "../detectionrules/sigma/tools/sigmac -t " + output + " --ignore-backend-errors " + file
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
(query, err) = p.communicate()
## Wait for date to terminate. Get return returncode ##
p_status = p.wait()
## have to remove '-' due to problems with Jinja2 variable naming,e.g es-qs throws error 'no es variable'
alert.update({output.replace("-", ""): str(query)[2:-3]})
###Data Needed
data_needed = main_dn_calculatoin_func(file)
alert.update({'data_needed': data_needed})
tactic = []
tactic_re = re.compile(r'attack\.\w\D+$')
technique = []
technique_re = re.compile(r'attack\.t\d{1,5}$')
other_tags = []
for tag in alert.get('tags'):
if tactic_re.match(tag):
tactic.append(ta_mapping.get(tag))
elif technique_re.match(tag):
technique.append(tag.upper()[7:])
else:
other_tags.append(tag)
alert.update({'tactics': tactic})
alert.update({'techniques': technique})
alert.update({'other_tags': other_tags})
triggers = []
for trigger in technique:
#trigger = re.search('t\d{1,5}', trigger).group(0).upper()
path = '../triggering/atomic-red-team/atomics/' + trigger + '/' + trigger + '.yaml'
try:
trigger_yaml = read_yaml_file(path)
triggers.append(trigger)
except FileNotFoundError:
print(trigger + ": No atomics trigger for this technique")
triggers.append(trigger +
": No atomics trigger for this technique")
alert.update({'description': alert.get('description').strip()})
alert.update({'triggers': triggers})
content = template.render(alert)
elif type == "loggingpolicy" or type == "LP":
template = env.get_template('markdown_loggingpolicy_template.md.j2')
parent_title = "Logging_Policies"
# get rid of newline to not mess with table in md
fields.update({'description': fields.get('description').strip()})
content = template.render(fields)
elif type == "dataneeded" or type == "DN":
template = env.get_template('markdown_dataneeded_template.md.j2')
parent_title = "Data_Needed"
fields.update({'description': fields.get('description').strip()})
content = template.render(fields)
elif type == "triggering" or type == "TG":
pass
else:
print("Unsuporrted type")
return
base = os.path.basename(file)
title = os.path.splitext(base)[0]
with open(
'../Atomic_Threat_Coverage/' + parent_title + "/" + title + ".md",
"w+") as fh:
fh.write(content)
def yaml2confluence_jinja(file, type, url, mail, password):
try:
import config # where we define confluence space name, list of DR and TG folders
space = config.confluence_space_name
list_of_detection_rules_directories = config.list_of_detection_rules_directories # not used so far
list_of_triggering_directories = config.list_of_triggering_directories # not used so far
confluence_name_of_root_directory = config.confluence_name_of_root_directory # not used so far
except:
space = "SOC"
pass
auth = HTTPBasicAuth(mail, password)
# there is a problem with working directory, for now this script must be run from scripts directory
env = Environment(loader=FileSystemLoader('templates'))
fields = read_yaml_file(file)
if type == "detectionrule" or type == "DR":
alert = fields
template = env.get_template('confluence_alert_template.html.j2')
parent_title = "Detection Rules"
sigma_rule = read_rule_file(file)
alert.update({'sigma_rule': sigma_rule})
outputs = ["es-qs", "xpack-watcher", "graylog"]
for output in outputs:
cmd = "../detectionrules/sigma/tools/sigmac -t " + output + " --ignore-backend-errors " + file
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
(query, err) = p.communicate()
## Wait for date to terminate. Get return returncode ##
p_status = p.wait()
## have to remove '-' due to problems with Jinja2 variable naming,e.g es-qs throws error 'no es variable'
alert.update({output.replace("-", ""): str(query)[2:-3]})
###Data Needed
data_needed = main_dn_calculatoin_func(file)
data_needed_with_id = []
for data in data_needed:
data_needed_id = str(get_page_id(url, auth, space, data))
data = (data, data_needed_id)
data_needed_with_id.append(data)
alert.update({'data_needed': data_needed_with_id})
tactic = []
tactic_re = re.compile(r'attack\.\w\D+$')
technique = []
technique_re = re.compile(r'attack\.t\d{1,5}$')
other_tags = []
for tag in alert.get('tags'):
if tactic_re.match(tag):
tactic.append(ta_mapping.get(tag))
elif technique_re.match(tag):
technique.append(tag.upper()[7:])
else:
other_tags.append(tag)
alert.update({'tactics': tactic})
alert.update({'techniques': technique})
alert.update({'other_tags': other_tags})
triggers = []
for trigger in technique:
#trigger = re.search('t\d{1,5}', trigger).group(0).upper()
path = '../triggering/atomic-red-team/atomics/' + trigger + '/' + trigger + '.yaml'
try:
trigger_yaml = read_yaml_file(path)
#main(path,'triggering')
trigger_id = str(get_page_id(url, auth, space, trigger))
trigger = (trigger, trigger_id)
print(trigger)
triggers.append(trigger)
except FileNotFoundError:
print(trigger + ": No atomics trigger for this technique")
alert.update({'triggers': triggers})
content = template.render(alert)
elif type == "loggingpolicy" or type == "LP":
template = env.get_template(
'confluence_loggingpolicy_template.html.j2')
parent_title = "Logging Policies"
content = template.render(fields)
elif type == "dataneeded" or type == "DN":
template = env.get_template('confluence_dataneeded_template.html.j2')
parent_title = "Data Needed"
logging_policies = fields.get("loggingpolicy")
logging_policies_with_id = []
for lp in logging_policies:
logging_policies_id = str(get_page_id(url, auth, space, lp))
lp = (lp, logging_policies_id)
logging_policies_with_id.append(lp)
fields.update({'loggingpolicy': logging_policies_with_id})
content = template.render(fields)
elif type == "triggering" or type == "TG":
template = env.get_template('confluence_trigger_template.html.j2')
parent_title = "Triggering"
atomic_trigger = read_rule_file(file)
base = os.path.basename(file)
trigger = os.path.splitext(base)[0]
path_md = '../triggering/atomic-red-team/atomics/' + trigger + '/' + trigger + '.md'
with open(path_md, 'r') as myfile:
md_data = myfile.read()
fields.update({'atomic_trigger': atomic_trigger})
fields.update({'atomic_trigger_md': md_data})
content = template.render(fields)
else:
print("Unsuporrted type")
return
base = os.path.basename(file)
title = os.path.splitext(base)[0]
data = {
"title": title,
"spacekey": space,
"parentid": str(get_page_id(url, auth, space, parent_title)),
"confluencecontent": content,
}
# for debbugging purpouses
# with open("confluence_from_template.html", "w+") as fh:
# fh.write(content)
#print(push_to_confluence(data, url, auth))
push_to_confluence(data, url, auth)
print("done: " + base)
from utils import read_yaml_file
puniverse = read_yaml_file('puniverse.yaml')
def propose(member, action, expiry):
pass
def vote():
pass