def get(self):
log.info("Listing available anomalies")
from eci_worker import anomalies
import inspect
import ast
module_dir = os.path.join(os.getcwd(), 'eci_worker')
module_file = os.path.join(module_dir, 'anomalies.py')
with open(module_file) as fd:
file_content = fd.read()
module = ast.parse(file_content)
function_definitions = [
node for node in module.body if isinstance(node, ast.FunctionDef)
]
list_function_definition = [f.name for f in function_definitions]
anomaly_list = []
for f in function_definitions:
anomaly_detail = {}
ds = ast.get_docstring(f)
if ds is None:
pass
else:
# All functions whos docstring starts with ECI& is considered as an anomaly inducer
if 'ECI' == ds.split('&')[0]:
# print(f.name)
anomaly_detail[f.name] = ds
anomaly_list.append(anomaly_detail)
# all_functions = inspect.getmembers(anomalies, inspect.isfunction)
# for fn in all_functions:
# # print(fn[0])
# print(fn[1].__doc__)
# # string = fn[1].__doc__
# # print(string.split('&'))
return jsonify({'anomalies': anomaly_list})
def delete(self):
if not request.json:
list_workers = get_list_workers()
for worker in range(len(list_workers)):
service_name = "chaosrqworker@{}".format(worker)
subprocess.call(['sudo', 'service', service_name, 'stop'])
time.sleep(1)
log.info("Stopped all chaos workers")
response = jsonify({'status': 'stopped all chaos workers'})
response.status_code = 200
elif 'id' not in request.json.keys():
response = jsonify({'status': 'missing worker id in request'})
response.status_code = 400
log.error("No worker id for stopping specified")
else:
service_name = "chaosrqworker@{}".format(request.json['id'])
subprocess.call(['sudo', 'service', service_name, 'stop'])
time.sleep(1)
log.info("Worker with id {} stopped".format(request.json['id']))
response = jsonify({
'status':
'stopped worker with id {}'.format(request.json['id'])
})
response.status_code = 200
return response
def put(self):
if not request.is_json:
response = jsonify({"error": 'session must be JSON'})
response.status_code = 415
return response
chaosgen.define_session(request.json)
log.info("New session definition received")
response = jsonify(request.json)
response.status_code = 201
return response
def remove_job(self, job_id):
try:
job = Job.fetch(job_id, connection=self.r_connection)
except Exception as inst:
log.error("No job with id {}".format(job_id))
response = {'error': 'no such job'}
return response
job.delete()
log.info("Job with id {} deleted.".format(job_id))
response = {"status": "deleted",
"job": job_id}
return response
def remove_all_jobs(self):
jobs = self.queue.get_job_ids()
for rjob in jobs:
try:
job = Job.fetch(rjob, connection=self.r_connection)
job.delete()
log.info("Deleted job with id {}".format(rjob))
except:
log.error("No job with id {}".format(rjob))
response = {'error': 'no such job'}
return response
response = {'deleted_job': jobs}
return response
def job_gen(self):
if not self.schedueled_session:
jobs = self.session_gen()
else:
jobs = self.schedueled_session
mod = import_module(self.anomaly_def)
# print(mod)
log.info("Defined jobs: {}".format(jobs))
for rjob in jobs:
ano_inst = getattr(mod, rjob['type'])
job = self.queue.enqueue(ano_inst, **rjob['options'])
id = job.get_id()
# print(ano_inst)
log.info("Queued anomaly {} with params {} and id {}".format(rjob['type'], rjob['options'], id))
self.detailed_session_info[id] = {'anomaly': rjob['type'], 'options': rjob['options']}
self.schedueled.append(id)
def post(self):
list_workers = get_list_workers()
logic_cpu = psutil.cpu_count(logical=True)
if len(
list_workers
) > logic_cpu - 1: # todo fix issue with pid file remaining after shutdown, can't start new workers.
response = jsonify({
'warning': 'maximum number of workers active!',
'workers': logic_cpu
})
log.warning('Maximum number of chaos workers reached: {}'.format(
logic_cpu))
response.status_code = 200
return response
service_name = "chaosrqworker@{}".format(len(list_workers))
subprocess.call(['sudo', 'service', service_name, 'start'])
log.info("Starting chaos worker {}".format(len(list_workers)))
response = jsonify({'status': 'workers started'})
response.status_code = 201
return response
def session_gen(self):
sgen = self.session
log.info("Detailed session generation started")
options = sgen.get('options', None)
anomalies = sgen.get('anomalies', None)
if options is not None:
self.loop = options.get('loop', False)
self.randomise = options.get('randomise', False)
self.distribution = options.get('distribution', False)
self.sample_size = options.get('sample_size', len(anomalies))
self.stagger_time = options.get('stagger_time', 0)
# print(anomalies)
if self.randomise:
log.info("Session is randomised")
random.shuffle(anomalies)
# sgen = anomalies
if self.distribution == 'uniform':
log.info("Session uniform distribution")
sgen = list(np.random.choice(anomalies, self.sample_size))
# print(len(np.random.choice(anomalies, self.sample_size)))
elif self.distribution == 'prob':
log.info("Session probabilistic distribution")
prob_list = []
for ano in anomalies:
prob_list.append(ano['prob'])
sum_prob = sum(prob_list)
if round(sum_prob, 2) < 1.0 or round(sum_prob, 2) > 1.0:
log.error("Probabilities sum to {}, have to be 1.0".format(sum_prob))
import sys
sys.exit()
sgen = list(np.random.choice(anomalies, self.sample_size, p=prob_list))
else:
sgen = anomalies
if self.stagger_time:
log.info("Stagger set to {}".format(self.stagger_time))
stagger = {'type': 'dummy', 'options': {'stime': self.stagger_time, 'silent': 1}}
sgen = self._intersperse(sgen=sgen, item=stagger)
# print(anomalies)
# print(len(anomalies))
# print(options)
self.schedueled_session = sgen
log.info("Finished final session generation")
return sgen
def define_session(self, session):
log.info("New session raw defined")
self.session = session