def ospf_nbr_down(host, yang_message, error, tag, process_number,
current_case):
'''
Once this function is called it has already been determined that a specific OSPF
process needs to be restarted. Most of the data gathering happened earlier.
Will attempt to restart the ospf process on the affected device. Last step
is a check to see if all the neighbors triggered OSPF_NEIGHBOR_UP events.
:param host: the host that started this workflow.
:param yang_message: the yang data.
:param error: the events error.
:param tag: the events tag.
:param current_case: the current oats case id.
:return: error, tag, a comment, configuration changes, the slack post satus (bool), success (bool).
'''
pool = ThreadPool(processes=1)
comment = 'OSPF neighbor down status on host {0} detected.'.format(host)
if current_case is None or current_case == 'None':
current_case = oatspsql.create_case(
error,
host,
solution='Case created in salt: `tshoot.ospf_nbr_down`.')
interface = oatsdbhelpers.get_interface(error, yang_message)
interface_neighbor = oatsnb.get_interface_neighbor(host,
interface,
case=current_case)
n_of_neighbors = len(
oatsnb.get_ospf_neighbors(interface_neighbor, case=current_case))
def test_cases_of_last_day():
caseid = oatspsql.create_case(error='test',
host='test',
description='Test',
status='new')
cases = oatspsql.show_cases_of_last_day()
deleted = oatspsql.delete_case(caseid)
assert caseid is not None and len(cases) >= 1 and deleted is True
def test_take_case():
caseid = oatspsql.create_case(error='test',
host='test',
description='Test',
status='new')
tech_id = oatspsql.take_case(case_id=caseid, technician='techy')
deleted = oatspsql.delete_case(caseid)
assert caseid is not None and tech_id is not None and deleted is True
def test_update_case():
caseid = oatspsql.create_case(error='test',
host='test',
description='Test',
status='new')
updated = oatspsql.update_case(caseid, solution='New stuff')
deleted = oatspsql.delete_case(caseid)
assert updated is not None and caseid is not None and deleted is True
def test_create_and_delete_case():
caseid = None
caseid = oatspsql.create_case(error='test',
host='test',
description='Test',
status='new')
deleted = oatspsql.delete_case(caseid)
assert deleted is True and caseid is not None
def test_open_cases():
caseid = oatspsql.create_case(error='test',
host='test',
description='Test',
status='new')
cases = oatspsql.numb_open_cases()
deleted = oatspsql.delete_case(caseid)
assert caseid is not None and cases >= 1 and deleted is True
def port_flap(host, yang_message, error, tag, current_case=None):
'''
Workflow for fixing port flapping, similarly to the ifdown function.
:param host: the from which the event originated.
:param yang_message: the yang data.
:param error: the events error.
:param tag: the events tag.
:param current_case: the current oats case id.
:return: error, tag, comment, success (bool)
'''
conf = 'No changes'
success = False
interface = oatsdbhelpers.get_interface(error, yang_message)
comment = 'Port Flapping on ' + host + ' detected. '
if current_case is None or current_case == 'None':
current_case = oatspsql.create_case(
error, host, solution='Case created in salt: `tshoot.port_flap`.')
interface_neighbor = oatsnb.get_interface_neighbor(host,
interface,
case=current_case)
#neighbors = oatsnb.get_neighbors(interface_neighbor, case=current_case)
#device_up = oatssalthelpers.check_device_connectivity(neighbors, interface_neighbor, case=current_case)
oatssalthelpers.if_shutdown(host, interface, case=current_case)
conf = oatssalthelpers.if_noshutdown(host, interface, case=current_case)
success = oatssalthelpers.ping(host,
interface_neighbor,
check_connectivity=True,
case=current_case)
if success:
success = True
comment += ('Resolved Port Flapping on Interface`' + interface + '`.')
oatssalthelpers.post_slack(comment, case=current_case)
oatspsql.close_case(current_case)
else:
oatspsql.update_case(
current_case,
solution='Port flapping could not get resolved. Technician needed.',
status=oatspsql.Status.ONHOLD.value)
comment = ('Could not fix port flapping status of `' + interface +
'` on host' + host + ' .')
oatssalthelpers.post_slack(comment, case=current_case)
success = False
oatspsql.update_case(current_case,
solution='Device ' + interface_neighbor +
' is unreachable. Technician needed.',
status=oatspsql.Status.ONHOLD.value)
oatssalthelpers.post_slack(comment, case=current_case)
return {
'error': error,
'tag': tag,
'comment': comment,
'changes': conf,
'success': success
}
def test_get_solutions_as_string():
caseid = oatspsql.create_case(error='test',
host='test',
description='Test',
status='new')
sol = []
sol = oatspsql.get_solutions_as_string(caseid)
print sol
deleted = oatspsql.delete_case(caseid)
assert sol is not None and caseid is not None and deleted is True
def out_discards_exceeded(data, host, timestamp, current_case):
'''
Function that loads a policy onto a device affected by a high amount of discarded packets.
The policy throttles the traffic from one IP to another with a certain port number.
The source-, destination-IP and port number are gathered by evaluating netflow data.
After a certain amount of time the policy is removed again.
:param data: contains the affected interface and the value of discarded packets.
:param host: the affected host.
:param timestamp: the timestamp of the event.
:param current_case: the current oats case-id
:return: error, comment, changes, slack-post-status(bool), success(bool)
'''
if current_case is None or current_case == 'None':
current_case = oatspsql.create_case(
"OUT_DISCARDS_EXCEEDED",
host,
solution='Case created in salt: `tshoot.out_discards_exceeded`.')
src_flow = None
# timeout while loop after 20secs
timeout = time.time() + 60
comment = ''
while src_flow is None:
flows = oatsinflux.get_type_data('netflow',
timestamp,
'netflow/*/data',
30,
host=host)
src_flow = oatssalthelpers.get_src_flow(flows, 15000)
time.sleep(1)
if time.time() > timeout:
break
if src_flow is not None:
dst_flow_port = src_flow['11']
interface = data['name']
src_ip_address = src_flow['8']
dst_ip_address = src_flow['12']
oatspsql.update_case(
current_case,
solution=
'Found responsible flow: src_ip = `{0}`, dst_ip = `{1}`, port_number = `{2}`'
.format(src_ip_address, dst_ip_address, dst_flow_port))
minion = oatsnb.get_hostname(host)
oatssalthelpers.apply_policy(minion, 8000, interface, src_ip_address,
dst_ip_address, dst_flow_port)
oatssalthelpers.remove_policy(minion, interface, src_ip_address,
dst_ip_address, dst_flow_port)
comment = "Discarded packets on host {0} on egress interface `{1}` exceeded threshhold. " \
"Destination port of traffic: `{2}`.\n".format(host, data['name'], dst_flow_port)
comment += "Applied traffic throttlinc policy for 120 seconds.\n"
else:
comment += 'Could not determine source of traffic, possible DDoS attack detected' \
' because traffic source port is `port 0`.'
slack_status = oatssalthelpers.post_slack(comment, case=current_case)
ret = {
'error': 'OUT_DISCARDS',
'comment': comment,
'changes': 'conf',
'slack-post-status:': slack_status,
'success': bool(dst_flow_port)
}
return ret
def ifdown(host, yang_message, error, tag, current_case=None):
'''
Function that gathers data in the network and executes a workflow according to the data.
Is triggered by the salt system once an INTERFACE_DOWN event arrives in the salt
event bus. Will try to fix the error or send a notification if it is unable to
do so.
:param host: the host that started this workflow.
:param yang_message: the yang data.
:param error: the events error.
:param tag: the events tag.
:param current_case: the current oats case id.
:return: error, tag, a comment, configuration changes, success (bool).
'''
conf = 'No changes'
success = False
interface = oatsdbhelpers.get_interface(error, yang_message)
comment = 'Interface down status on host ' + host + ' detected. '
if current_case is None or current_case == 'None':
current_case = oatspsql.create_case(
error, host, solution='Case created in salt: `tshoot.ifdown`.')
interface_neighbor = oatsnb.get_interface_neighbor(host,
interface,
case=current_case)
neighbors = oatsnb.get_neighbors(interface_neighbor, case=current_case)
device_up = oatssalthelpers.check_device_connectivity(neighbors,
interface_neighbor,
case=current_case)
if device_up:
# cycle affected interface
oatssalthelpers.if_shutdown(host, interface, case=current_case)
conf = oatssalthelpers.if_noshutdown(host,
interface,
case=current_case)
# check if cycle was successful
success = oatssalthelpers.ping(host,
interface_neighbor,
check_connectivity=True,
case=current_case)
if success:
success = True
comment += ('Config for Interface `' + interface +
'` automatically changed from down to up')
# TODO: remove, only useful for debugging
oatssalthelpers.post_slack(comment, case=current_case)
oatspsql.close_case(current_case)
else:
oatspsql.update_case(current_case,
solution=error +
'could not get resolved. Technician needed.',
status=oatspsql.Status.ONHOLD.value)
comment = ('Could not fix down status of `' + interface +
'` on host' + host + ' .')
oatssalthelpers.post_slack(comment, case=current_case)
if not device_up:
# TODO: powercycle, check power consumation
success = False
oatspsql.update_case(current_case,
solution='Device ' + interface_neighbor +
' is unreachable. Technician needed.',
status=oatspsql.Status.ONHOLD.value)
comment += 'Interface `' + interface + '` on host ' + host + ' down. Neighbor ' + interface_neighbor + ' is down.'
oatssalthelpers.post_slack(comment, case=current_case)
comment += ' Could not restore connectivity - Slack Message sent.'
return {
'error': error,
'tag': tag,
'comment': comment,
'changes': conf,
'success': success
}
def __create_db_case(error, host, function_name):
return oatspsql.create_case(error, host,
solution='Case started in kafka event consumer: `correlate.' + function_name + '`')