def run(self):
edn_pid = os.fork()
if edn_pid == 0:
try:
while True:
item = InternalServer.get_dagda_edn().get()
if item['msg'] == 'init_db':
self._init_or_update_db()
elif item['msg'] == 'check_image':
self._check_docker_by_image_name(item)
elif item['msg'] == 'check_container':
self._check_docker_by_container_id(item)
except KeyboardInterrupt:
# Pressed CTRL+C to quit, so nothing to do
pass
else:
sysdig_falco_monitor_pid = os.fork()
if sysdig_falco_monitor_pid == 0:
try:
self.sysdig_falco_monitor.pre_check()
self.sysdig_falco_monitor.run()
except DagdaError as e:
DagdaLogger.get_logger().error(e.get_message())
DagdaLogger.get_logger().warning(
'Runtime behaviour monitor disabled.')
except KeyboardInterrupt:
# Pressed CTRL+C to quit
InternalServer.get_docker_driver().docker_stop(
self.sysdig_falco_monitor.get_running_container_id())
InternalServer.get_docker_driver().docker_remove_container(
self.sysdig_falco_monitor.get_running_container_id())
else:
DagdaServer.app.run(debug=False,
host=self.dagda_server_host,
port=self.dagda_server_port)
def run(self):
edn_pid = os.fork()
if edn_pid == 0:
try:
while True:
item = InternalServer.get_dagda_edn().get()
if item['msg'] == 'init_db':
self._init_or_update_db()
elif item['msg'] == 'check_image':
self._check_docker_by_image_name(item)
elif item['msg'] == 'check_container':
self._check_docker_by_container_id(item)
except KeyboardInterrupt:
# Pressed CTRL+C to quit, so nothing to do
None
else:
sysdig_falco_monitor_pid = os.fork()
if sysdig_falco_monitor_pid == 0:
try:
self.sysdig_falco_monitor.pre_check()
self.sysdig_falco_monitor.run()
except KeyboardInterrupt:
# Pressed CTRL+C to quit
InternalServer.get_docker_driver().docker_stop(self.sysdig_falco_monitor.get_running_container_id())
else:
DagdaServer.app.run(debug=False, host=self.dagda_server_host, port=self.dagda_server_port)
def check_docker_by_image_name(image_name):
# -- Check input
if not image_name:
return json.dumps({
'err': 400,
'msg': 'Bad image name'
},
sort_keys=True), 400
# -- Docker pull from remote registry if it is necessary
try:
pulled = False
if not InternalServer.get_docker_driver().is_docker_image(image_name):
if ':' in image_name:
tmp = image_name.split(':')[0]
tag = image_name.split(':')[1]
msg = 'Error: image library/' + image_name + ':' + tag + ' not found'
output = InternalServer.get_docker_driver().docker_pull(
tmp, tag=tag)
else:
msg = 'Error: image library/' + image_name + ':latest not found'
output = InternalServer.get_docker_driver().docker_pull(
image_name)
if 'errorDetail' in output:
DagdaLogger.get_logger().error(msg)
raise DagdaError(msg)
pulled = True
except Exception as ex:
message = "Unexpected exception of type {0} occurred while pulling the docker image: {1!r}" \
.format(type(ex).__name__, ex.get_message() if type(ex).__name__ == 'DagdaError' else ex.args)
DagdaLogger.get_logger().error(message)
return json.dumps({
'err': 404,
'msg': 'Image name not found'
},
sort_keys=True), 404
# -- Process request
data = {}
data['image_name'] = image_name
data['timestamp'] = datetime.datetime.now().timestamp()
data['status'] = 'Analyzing'
id = InternalServer.get_mongodb_driver(
).insert_docker_image_scan_result_to_history(data)
InternalServer.get_dagda_edn().put({
'msg': 'check_image',
'image_name': image_name,
'_id': str(id),
'pulled': pulled
})
# -- Return
output = {}
output['id'] = str(id)
output['msg'] = 'Accepted the analysis of <' + image_name + '>'
return json.dumps(output, sort_keys=True), 202
def _check_docker_by_image_name(item):
analyzer = Analyzer()
# -- Evaluates the docker image
evaluated_docker_image = analyzer.evaluate_image(item['image_name'], None)
# -- Updates mongodb report
InternalServer.get_mongodb_driver().update_docker_image_scan_result_to_history(item['_id'],
evaluated_docker_image)
# -- Cleanup
if item['pulled']:
InternalServer.get_docker_driver().docker_remove_image(item['image_name'])
def _check_docker_by_image_name(item):
analyzer = Analyzer()
# -- Evaluates the docker image
evaluated_docker_image = analyzer.evaluate_image(item['image_name'], None)
# -- Updates mongodb report
InternalServer.get_mongodb_driver().update_docker_image_scan_result_to_history(item['_id'],
evaluated_docker_image)
# -- Cleanup
if item['pulled']:
InternalServer.get_docker_driver().docker_remove_image(item['image_name'])
def check_docker_by_container_id(container_id):
# -- Check input
if not container_id:
return json.dumps({'err': 400, 'msg': 'Bad container id'}, sort_keys=True), 400
# -- Retrieves docker image name
try:
image_name = InternalServer.get_docker_driver().get_docker_image_name_by_container_id(container_id)
except Exception as ex:
message = "Unexpected exception of type {0} occurred while getting the docker image name: {1!r}" \
.format(type(ex).__name__, ex.get_message() if type(ex).__name__ == 'DagdaError' else ex.args)
DagdaLogger.get_logger().error(message)
return json.dumps({'err': 404, 'msg': 'Container Id not found'}, sort_keys=True), 404
# -- Process request
data = {}
data['image_name'] = image_name
data['timestamp'] = datetime.datetime.now().timestamp()
data['status'] = 'Analyzing'
id = InternalServer.get_mongodb_driver().insert_docker_image_scan_result_to_history(data)
InternalServer.get_dagda_edn().put({'msg': 'check_container', 'container_id': container_id, '_id': str(id)})
# -- Return
output = {}
output['id'] = str(id)
output['msg'] = 'Accepted the analysis of <' + image_name + '> with id: ' + container_id
return json.dumps(output, sort_keys=True), 202
def stop_monitor_by_container_id(container_id):
# -- Check runtime monitor status
if not InternalServer.is_runtime_analysis_enabled():
return json.dumps({'err': 503, 'msg': 'Behaviour analysis service unavailable'}, sort_keys=True), 503
# -- Checks input
if not container_id:
return json.dumps({'err': 400, 'msg': 'Bad container id'}, sort_keys=True), 400
# -- Retrieves docker image name
try:
image_name = InternalServer.get_docker_driver().get_docker_image_name_by_container_id(container_id)
except:
return json.dumps({'err': 404, 'msg': 'Container Id not found'}, sort_keys=True), 404
# -- Checks if the container is already being monitoring
if not InternalServer.get_mongodb_driver().is_there_a_started_monitoring(container_id):
return json.dumps({'err': 400, 'msg': 'There is not monitoring for the requested container id'},
sort_keys=True), 400
now = datetime.datetime.now().timestamp()
# -- Process request
InternalServer.get_mongodb_driver().update_runtime_monitoring_analysis(container_id)
monitoring_result = InternalServer.get_mongodb_driver().get_a_started_monitoring(container_id)
monitoring_result['runtime_analysis']['stop_timestamp'] = now
monitoring_result['status'] = 'Completed'
id = str(monitoring_result['_id'])
# -- Update history
InternalServer.get_mongodb_driver().update_docker_image_scan_result_to_history(id, monitoring_result)
# -- Return
return json.dumps(InternalServer.get_mongodb_driver().get_docker_image_history(image_name, id)[0], sort_keys=True)
def stop_monitor_by_container_id(container_id):
# -- Checks input
if not container_id:
return json.dumps({'err': 400, 'msg': 'Bad container id'}, sort_keys=True), 400
# -- Retrieves docker image name
try:
image_name = InternalServer.get_docker_driver().get_docker_image_name_from_container_id(container_id)
except:
return json.dumps({'err': 404, 'msg': 'Container Id not found'}, sort_keys=True), 404
# -- Checks if the container is already being monitoring
if not InternalServer.get_mongodb_driver().is_there_a_started_monitoring(container_id):
return json.dumps({'err': 400, 'msg': 'There is not monitoring for the requested container id'},
sort_keys=True), 400
now = datetime.datetime.now().timestamp()
# -- Process request
InternalServer.get_mongodb_driver().update_runtime_monitoring_analysis(container_id)
monitoring_result = InternalServer.get_mongodb_driver().get_a_started_monitoring(container_id)
monitoring_result['runtime_analysis']['stop_timestamp'] = now
monitoring_result['status'] = 'Completed'
id = str(monitoring_result['_id'])
# -- Update history
InternalServer.get_mongodb_driver().update_docker_image_scan_result_to_history(id, monitoring_result)
# -- Return
return json.dumps(InternalServer.get_mongodb_driver().get_docker_image_history(image_name, id)[0], sort_keys=True)
def start_monitor_by_container_id(container_id):
# -- Checks input
if not container_id:
return json.dumps({'err': 400, 'msg': 'Bad container id'}, sort_keys=True), 400
# -- Retrieves docker image name
try:
image_name = InternalServer.get_docker_driver().get_docker_image_name_from_container_id(container_id)
except:
return json.dumps({'err': 404, 'msg': 'Container Id not found'}, sort_keys=True), 404
# -- Checks if the container is already being monitoring
if InternalServer.get_mongodb_driver().is_there_a_started_monitoring(container_id):
return json.dumps({'err': 400, 'msg': 'The monitoring for the requested container id is already started'},
sort_keys=True), 400
now = datetime.datetime.now().timestamp()
# -- Create image_history
history = {}
history['image_name'] = image_name
history['timestamp'] = now
history['status'] = 'Monitoring'
history['runtime_analysis'] = {'container_id': container_id,
'start_timestamp': now,
'stop_timestamp': None,
'anomalous_activities_detected': None}
id = InternalServer.get_mongodb_driver().insert_docker_image_scan_result_to_history(history)
# -- Return
output = {}
output['id'] = str(id)
output['image_name'] = image_name
output['msg'] = 'Monitoring of docker container with id <' + container_id + '> started'
return json.dumps(output, sort_keys=True), 202
def __init__(self, dagda_server_host='127.0.0.1', dagda_server_port=5000, mongodb_host='127.0.0.1',
mongodb_port=27017):
super(DagdaServer, self).__init__()
self.dagda_server_host = dagda_server_host
self.dagda_server_port = dagda_server_port
InternalServer.set_mongodb_driver(mongodb_host, mongodb_port)
self.sysdig_falco_monitor = SysdigFalcoMonitor(InternalServer.get_docker_driver(),
InternalServer.get_mongodb_driver())
def check_docker_by_image_name(image_name):
# -- Check input
if not image_name:
return json.dumps({
'err': 400,
'msg': 'Bad image name'
},
sort_keys=True), 400
# -- Docker pull from remote registry if it is necessary
try:
pulled = False
if not InternalServer.get_docker_driver().is_docker_image(image_name):
output = InternalServer.get_docker_driver().docker_pull(image_name)
if 'errorDetail' in output:
msg = 'Error: image library/' + image_name + ':latest not found'
DagdaLogger.get_logger().error(msg)
raise DagdaError(msg)
pulled = True
except:
return json.dumps({
'err': 404,
'msg': 'Image name not found'
},
sort_keys=True), 404
# -- Process request
data = {}
data['image_name'] = image_name
data['timestamp'] = datetime.datetime.now().timestamp()
data['status'] = 'Analyzing'
id = InternalServer.get_mongodb_driver(
).insert_docker_image_scan_result_to_history(data)
InternalServer.get_dagda_edn().put({
'msg': 'check_image',
'image_name': image_name,
'_id': str(id),
'pulled': pulled
})
# -- Return
output = {}
output['id'] = str(id)
output['msg'] = 'Accepted the analysis of <' + image_name + '>'
return json.dumps(output, sort_keys=True), 202
def __init__(self, dagda_server_url=None):
super(Analyzer, self).__init__()
self.is_remote = False
if dagda_server_url is not None:
self.dagda_server_url = dagda_server_url
self.is_remote = True
else:
self.mongoDbDriver = InternalServer.get_mongodb_driver()
self.dockerDriver = InternalServer.get_docker_driver()
def __init__(self, dagda_server_url=None):
super(Analyzer, self).__init__()
self.is_remote = False
if dagda_server_url is not None:
self.dagda_server_url = dagda_server_url
self.is_remote = True
else:
self.mongoDbDriver = InternalServer.get_mongodb_driver()
self.dockerDriver = InternalServer.get_docker_driver()
def __init__(self, dagda_server_host='127.0.0.1', dagda_server_port=5000, mongodb_host='127.0.0.1',
mongodb_port=27017, mongodb_ssl=False, mongodb_user=None, mongodb_pass=None,
falco_rules_filename=None):
super(DagdaServer, self).__init__()
self.dagda_server_host = dagda_server_host
self.dagda_server_port = dagda_server_port
InternalServer.set_mongodb_driver(mongodb_host, mongodb_port, mongodb_ssl, mongodb_user, mongodb_pass)
self.sysdig_falco_monitor = SysdigFalcoMonitor(InternalServer.get_docker_driver(),
InternalServer.get_mongodb_driver(),
falco_rules_filename)
def run(self):
edn_pid = os.fork()
if edn_pid == 0:
try:
while True:
item = InternalServer.get_dagda_edn().get()
if item['msg'] == 'init_db':
self._init_or_update_db()
elif item['msg'] == 'check_image':
self._check_docker_by_image_name(item)
elif item['msg'] == 'check_container':
self._check_docker_by_container_id(item)
except KeyboardInterrupt:
# Pressed CTRL+C to quit, so nothing to do
pass
else:
docker_events_monitor_pid = os.fork()
if docker_events_monitor_pid == 0:
try:
docker_daemon_events_monitor = DockerDaemonEventsMonitor(InternalServer.get_docker_driver(),
InternalServer.get_mongodb_driver())
docker_daemon_events_monitor.run()
except KeyboardInterrupt:
# Pressed CTRL+C to quit, so nothing to do
pass
else:
sysdig_falco_monitor_pid = os.fork()
if sysdig_falco_monitor_pid == 0:
try:
self.sysdig_falco_monitor.pre_check()
self.sysdig_falco_monitor.run()
except DagdaError as e:
DagdaLogger.get_logger().error(e.get_message())
DagdaLogger.get_logger().warning('Runtime behaviour monitor disabled.')
except KeyboardInterrupt:
# Pressed CTRL+C to quit
if not InternalServer.is_external_falco():
InternalServer.get_docker_driver().docker_stop(self.sysdig_falco_monitor.get_running_container_id())
InternalServer.get_docker_driver().docker_remove_container(
self.sysdig_falco_monitor.get_running_container_id())
else:
DagdaServer.app.run(debug=False, host=self.dagda_server_host, port=self.dagda_server_port)
def check_docker_by_image_name(image_name):
# -- Check input
if not image_name:
return json.dumps({'err': 400, 'msg': 'Bad image name'}, sort_keys=True), 400
# -- Docker pull from remote registry if it is necessary
try:
pulled = False
if not InternalServer.get_docker_driver().is_docker_image(image_name):
if ':' in image_name:
tmp = image_name.split(':')[0]
tag = image_name.split(':')[1]
msg = 'Error: image library/' + image_name + ':' + tag + ' not found'
output = InternalServer.get_docker_driver().docker_pull(tmp, tag=tag)
else:
msg = 'Error: image library/' + image_name + ':latest not found'
output = InternalServer.get_docker_driver().docker_pull(image_name)
if 'errorDetail' in output:
DagdaLogger.get_logger().error(msg)
raise DagdaError(msg)
pulled = True
except Exception as ex:
message = "Unexpected exception of type {0} occurred while pulling the docker image: {1!r}" \
.format(type(ex).__name__, ex.get_message() if type(ex).__name__ == 'DagdaError' else ex.args)
DagdaLogger.get_logger().error(message)
return json.dumps({'err': 404, 'msg': 'Image name not found'}, sort_keys=True), 404
# -- Process request
data = {}
data['image_name'] = image_name
data['timestamp'] = datetime.datetime.now().timestamp()
data['status'] = 'Analyzing'
id = InternalServer.get_mongodb_driver().insert_docker_image_scan_result_to_history(data)
InternalServer.get_dagda_edn().put({'msg': 'check_image', 'image_name': image_name, '_id': str(id),
'pulled': pulled})
# -- Return
output = {}
output['id'] = str(id)
output['msg'] = 'Accepted the analysis of <' + image_name + '>'
return json.dumps(output, sort_keys=True), 202
def __init__(self, dagda_server_host='127.0.0.1', dagda_server_port=5000, mongodb_host='127.0.0.1',
mongodb_port=27017, mongodb_ssl=False, mongodb_user=None, mongodb_pass=None,
falco_rules_filename=None, external_falco_output_filename=None, debug_logging=False):
super(DagdaServer, self).__init__()
self.dagda_server_host = dagda_server_host
self.dagda_server_port = dagda_server_port
InternalServer.set_debug_logging_enabled(debug_logging)
InternalServer.set_mongodb_driver(mongodb_host, mongodb_port, mongodb_ssl, mongodb_user, mongodb_pass)
self.sysdig_falco_monitor = SysdigFalcoMonitor(InternalServer.get_docker_driver(),
InternalServer.get_mongodb_driver(),
falco_rules_filename,
external_falco_output_filename)
def get_all_running_containers():
containers = InternalServer.get_docker_driver().get_docker_client().containers()
output = []
for container in containers:
c = {}
c['id'] = container['Id'][:12]
c['image'] = container['Image']
c['created'] = str(datetime.datetime.utcfromtimestamp(container['Created']))
c['status'] = container['State']
c['name'] = container['Names'][0][1:]
output.append(c)
return json.dumps(output, sort_keys=True)
def get_all_docker_images():
images = InternalServer.get_docker_driver().get_docker_client().images()
output = []
for image in images:
i = {}
if image['RepoTags'] is None:
i['tags'] = list(['None:None'])
else:
i['tags'] = list(set(image['RepoTags']))
i['id'] = image['Id'][7:][:12]
i['created'] = str(datetime.datetime.utcfromtimestamp(image['Created']))
i['size'] = sizeof_fmt(image['VirtualSize'])
output.append(i)
return json.dumps(output, sort_keys=True)
def check_docker_by_container_id(container_id):
# -- Check input
if not container_id:
return json.dumps({
'err': 400,
'msg': 'Bad container id'
},
sort_keys=True), 400
# -- Retrieves docker image name
try:
image_name = InternalServer.get_docker_driver(
).get_docker_image_name_by_container_id(container_id)
except Exception as ex:
message = "Unexpected exception of type {0} occurred while getting the docker image name: {1!r}" \
.format(type(ex).__name__, ex.get_message() if type(ex).__name__ == 'DagdaError' else ex.args)
DagdaLogger.get_logger().error(message)
return json.dumps({
'err': 404,
'msg': 'Container Id not found'
},
sort_keys=True), 404
# -- Process request
data = {}
data['image_name'] = image_name
data['timestamp'] = datetime.datetime.now().timestamp()
data['status'] = 'Analyzing'
id = InternalServer.get_mongodb_driver(
).insert_docker_image_scan_result_to_history(data)
InternalServer.get_dagda_edn().put({
'msg': 'check_container',
'container_id': container_id,
'_id': str(id)
})
# -- Return
output = {}
output['id'] = str(id)
output[
'msg'] = 'Accepted the analysis of <' + image_name + '> with id: ' + container_id
return json.dumps(output, sort_keys=True), 202
def check_docker_by_container_id(container_id):
# -- Check input
if not container_id:
return json.dumps({
'err': 400,
'msg': 'Bad container id'
},
sort_keys=True), 400
# -- Retrieves docker image name
try:
image_name = InternalServer.get_docker_driver(
).get_docker_image_name_from_container_id(container_id)
except:
return json.dumps({
'err': 404,
'msg': 'Container Id not found'
},
sort_keys=True), 404
# -- Process request
data = {}
data['image_name'] = image_name
data['timestamp'] = datetime.datetime.now().timestamp()
data['status'] = 'Analyzing'
_id = InternalServer.get_mongodb_driver(
).insert_docker_image_scan_result_to_history(data)
InternalServer.get_dagda_edn().put({
'msg': 'check_container',
'container_id': container_id,
'_id': str(_id)
})
# -- Return
output = {}
output['id'] = str(_id)
output[
'msg'] = 'Accepted the analysis of <' + image_name + '> with id: ' + container_id
return json.dumps(output, sort_keys=True), 202
def start_monitor_by_container_id(container_id):
# -- Check runtime monitor status
if not InternalServer.is_runtime_analysis_enabled():
return json.dumps({'err': 503, 'msg': 'Behaviour analysis service unavailable'}, sort_keys=True), 503
# -- Checks input
if not container_id:
return json.dumps({'err': 400, 'msg': 'Bad container id'}, sort_keys=True), 400
# -- Retrieves docker image name
try:
image_name = InternalServer.get_docker_driver().get_docker_image_name_by_container_id(container_id)
except:
return json.dumps({'err': 404, 'msg': 'Container Id not found'}, sort_keys=True), 404
# -- Checks if the container is already being monitoring
if InternalServer.get_mongodb_driver().is_there_a_started_monitoring(container_id):
return json.dumps({'err': 400, 'msg': 'The monitoring for the requested container id is already started'},
sort_keys=True), 400
now = datetime.datetime.now().timestamp()
# -- Create image_history
history = {}
history['image_name'] = image_name
history['timestamp'] = now
history['status'] = 'Monitoring'
history['runtime_analysis'] = {'container_id': container_id,
'start_timestamp': now,
'stop_timestamp': None,
'anomalous_activities_detected': None}
id = InternalServer.get_mongodb_driver().insert_docker_image_scan_result_to_history(history)
# -- Return
output = {}
output['id'] = str(id)
output['image_name'] = image_name
output['msg'] = 'Monitoring of docker container with id <' + container_id + '> started'
return json.dumps(output, sort_keys=True), 202
def __init__(self):
super(Analyzer, self).__init__()
self.mongoDbDriver = InternalServer.get_mongodb_driver()
self.dockerDriver = InternalServer.get_docker_driver()
