def _change_test_status(self, **kwargs):
from salts.models import TestResult
from datetime import timedelta
shooting = kwargs.get('shooting')
start_time = time.time()
ctrl_c_delta = timedelta(seconds=TankManager.CTRL_C_INTERVAL)
curr_time = time.time()
while curr_time - start_time <= TankManager.WAIT_FOR_RESULT_SAVED:
try:
test_result = \
TestResult.objects.get(session_id=shooting.session_id)
except TestResult.DoesNotExist:
log.info("The test id=%s isn't "
"been saved yet." % shooting.session_id)
time.sleep(TankManager.POLL_INTERVAL)
curr_time = time.time()
continue
if test_result.dt_finish - test_result.dt_start >= ctrl_c_delta:
log.info("The test id=%s: "
"test duration exceeds 3 minutes, "
"the status remains Unknown." % shooting.session_id)
return
test_result.test_status = 'dbg'
test_result.save()
log.info("The test id=%s: "
"test duration less than 3 minutes, "
"the status is changed with Debug." % shooting.session_id)
return
log.warning("The test id=%s wasn't saved into DB." %
shooting.session_id)
def ini_files(dir_path):
ini = []
for root, dirs, files in os.walk(dir_path, topdown=False):
for name in files:
full_path = os.path.join(root, name)
file_name, file_ext = os.path.splitext(full_path)
if file_ext == '.ini':
config = ConfigParser()
try:
config.read(full_path)
except Error, exc:
log.warning("Config %s is not valid: %s" %
(full_path, exc))
else:
try:
test_name = config.get('salts_report', 'test_name')
except (NoSectionError, NoOptionError):
log.info("Config %s is not scenario: "
"there is not 'test_name' option "
"in the 'salts_report' section." % full_path)
else:
if test_name:
ini.append(re.sub("%s/" % dir_path, '', full_path))
else:
log.warning("Scenario %s contains "
"empty 'test_name' option." %
full_path)
def user_filter(request, results):
scenario_id = request_get_value(request, "scid")
scenario_path = request_get_value(request, "scpath")
target = request_get_value(request, "trg")
gen_type = request_get_value(request, "gt")
generator = request_get_value(request, "gen")
test_group = request_get_value(request, "tg")
test_search = request_get_value(request, "ts")
status = request_get_value(request, "st")
task_id = request_get_value(request, "tid")
spe = request_get_value(request, "spe")
from_time = request_get_value(request, "from")
to_time = request_get_value(request, "to")
rps_value = request_get_value(request, "rps")
if scenario_id:
try:
s = Scenario.objects.get(id=scenario_id)
scenario_path = s.scenario_path
except Scenario.DoesNotExist:
log.warning("The scid=%s parameter from request does not exist.",
scenario_id)
if scenario_path:
results = results.filter(scenario_path=scenario_path)
if target:
results = results.filter(target__contains=target)
if test_group:
results = results.filter(group=test_group)
if test_search:
results = results.filter(test_name__contains=test_search)
if gen_type:
results = results.filter(generator_types__name__contains=gen_type)
if generator:
results = results.filter(generator__contains=generator)
if rps_value:
results = results.filter(rps=rps_value)
if status:
status = status.split(",")
for (i, item) in enumerate(status):
if item == "unknown":
status[i] = u"unk"
if item == "debug":
status[i] = u"dbg"
results = results.filter(test_status__in=status)
if task_id:
results = results.filter(ticket_id=task_id)
if spe:
results = results.filter(user=spe)
if from_time:
from_time = from_time.replace("_", " ")
results = results.filter(dt_finish__gte=from_time)
if to_time:
to_time = to_time.replace("_", " ")
results = results.filter(dt_finish__lte=to_time)
return results
def obtain_tank(self, tank_id, reqdata):
try:
reqdata["tank"] = Tank.objects.filter(id=tank_id)
return None
except Tank.DoesNotExist:
resp = {'status': 'failed',
'message': "Given tank_id=%s isn't valid." \
% tank_id}
log.warning(resp['message'])
return HttpResponse(json.dumps(resp),
content_type="application/json",
status=404)
def obtain_scenario(self, scenario_id, reqdata):
try:
reqdata["scenario"] = Scenario.objects.filter(id=scenario_id)
return None
except Scenario.DoesNotExist:
resp = {'status': 'failed',
'message': "Given scenario_id=%s isn't valid." \
% scenario_id}
log.warning(resp['message'])
return HttpResponse(json.dumps(resp),
content_type="application/json",
status=404)
def _wait_for_status(self, client, session_id):
while True:
resp = client.status(session_id)
if "status" not in resp:
log.warning("Response doesn't contain the 'status' field. "
"Test with id=%s. Response: %s" %
(session_id, resp))
return resp
if resp["status"] == "running":
time.sleep(TankManager.WAIT_INTERVAL)
continue
return resp
def get_queryset(self):
host = self.request.query_params.get('host', None)
if not host:
return super(TankViewSet, self).get_queryset()
try:
info = socket.gethostbyname_ex(host)
except:
info = None
if not info:
log.warning("Invalid '%s' host name is given." % host)
return Tank.objects.none()
return Tank.objects.filter(host=info[0])
def check_auth(self, username, config):
if 'api_user' in config['salts']:
return None
if username:
config['salts']['api_user'] = username
return None
resp = {'status': 'failed',
'message': "User isn't authenticated " \
"or api_user option isn't provided."}
log.warning(resp['message'])
return HttpResponse(json.dumps(resp),
content_type="application/json",
status=401)
def get_used_for_web(self, scenario_path):
config = ConfigParser()
ini_path = os.path.join(self.dir_path, scenario_path)
if not os.path.exists(ini_path):
log.warning("Config %s is not found." % ini_path)
return False
config.read(ini_path)
if "salts" not in config.sections():
return True
if "used_for_web" not in config.options("salts"):
return True
bool_values = {"true": True, "false": False}
str_value = config.get("salts", "used_for_web").lower()
return bool_values.get(str_value, True)
def get_scenario_name(self, scenario_path):
config = ConfigParser()
ini_path = os.path.join(self.dir_path, scenario_path)
test_name = ''
if os.path.exists(ini_path):
config.read(ini_path)
try:
test_name = config.get(IniCtrl.REPORT_SECTION, 'test_name')
except (NoSectionError, NoOptionError):
log.warning("Config %s is not scenario: "
"there is not 'test_name' option "
"in the 'salts_report' section." % ini_path)
else:
log.warning("Config %s is not found." % ini_path)
return test_name
def run_test_api(request):
if not is_valid_request(request, ["tsid"]):
return HttpResponse("Request isn't valid.")
sess = pickle.loads(request.session["test_run"])
tsid = request.POST["tsid"]
if tsid in sess:
msg = "FUNC run_test_api: client with id=%s exist already." % tsid
log.warning(msg)
return HttpResponse(msg)
ts_record = TestSettings.objects.get(id=tsid)
path = os.path.join(LT_PATH, ts_record.file_path)
qs = TestSettings.objects.raw("SELECT g.id, g.host, g.port \
FROM salts_generator g \
JOIN salts_testsettings ts ON g.id = ts.generator_id WHERE ts.id = %s" % tsid)
(gen_id, host, port) = (qs[0].id, qs[0].host, qs[0].port)
client = TankClient(host, port, log)
sess[tsid] = {}
sess[tsid]["host"] = host
sess[tsid]["port"] = port
with open(path, "r") as ini_file:
tr = TestRun(generator_id=gen_id,
test_settings_id=tsid,
status=TestRun.STATUS_RUNNING)
tr.save()
resp = client.run(ini_file.read(), "start", tr.id)
if not resp:
msg = "No server response when test tried to start."
log.warning(msg)
return HttpResponse(msg)
else:
sess[tsid]["trid"] = tr.id
sess[tsid]["session"] = resp["session"]
sess[tsid]["wait_status"] = "prepare"
request.session["test_run"] = str(pickle.dumps(sess))
response_dict = {}
response_dict.update({"ini_path": path})
response_dict.update({"wait_status": "prepare"})
response_dict.update({"session": resp["session"]})
return HttpResponse(json.dumps(response_dict),
content_type="application/json")
def create(self, request, *args, **kwargs):
ex_data = {}
if request.META.get('HTTP_AUTHORIZATION'):
ex_data['token'] = \
request.META['HTTP_AUTHORIZATION'].replace('Token ', '')
self._add_ex_data(ex_data, request.data, 'force_run')
try:
serializer = self.get_serializer(data=request.data)
serializer.is_valid(raise_exception=True)
self.perform_create(serializer, **ex_data)
headers = self.get_success_headers(serializer.data)
return Response(serializer.data,
status=status.HTTP_201_CREATED,
headers=headers)
except ShootingHttpIssue, exc:
msg = "Shooting HTTP Issue: %s" % exc.message
log.warning(msg)
return Response(msg, status=exc.code, content_type='text/html')
def check_perm_start(self, config, scenario):
username = config['salts']['api_user']
cursor = connection.cursor()
cursor.execute("""
SELECT usr_gr.id FROM auth_user usr
JOIN auth_user_groups usr_gr ON usr.id = usr_gr.user_id
JOIN salts_scenario ss USING(group_id)
WHERE ss.id = {scenario_id} and usr.username = '******'
""".format(scenario_id=scenario.id, username=username))
if cursor.fetchone():
return None
resp = {'status': 'failed',
'message': "Test %s disabled for '%s' user." \
% (scenario.scenario_path,
config['salts']['api_user'])}
log.warning(resp['message'])
return HttpResponse(json.dumps(resp),
content_type="application/json",
status=403)
def check_perm_stop(self, username, shooting_id):
cursor = connection.cursor()
cursor.execute("""
SELECT sh.id FROM salts_shooting sh
JOIN auth_user_groups usr_gr USING(user_id)
JOIN auth_user_groups usr_gr_req
ON usr_gr_req.group_id=usr_gr.group_id
JOIN auth_user usr_req ON usr_req.id=usr_gr_req.user_id
WHERE sh.id={shooting_id} AND usr_req.username='******'
""".format(shooting_id=shooting_id, req_username=username))
if cursor.fetchone():
return None
resp = {'status': 'failed',
'message': "Shooting cannot be stopped by '%s' user." \
% username}
log.warning(resp['message'])
return HttpResponse(json.dumps(resp),
content_type="application/json",
status=403)
def adapt_tanks_list(self, tanks_list, active_shootings, request_user):
if request_user.id in self._actual_tanks_info:
for record in self._actual_tanks_info[request_user.id]:
sh = active_shootings.filter(id=record["shooting"]["id"])
if sh:
record["shooting"]["remained"] = remainedtime(sh[0])
else:
self._actual_tanks_info[request_user.id]["shooting"] = {}
else:
self._actual_tanks_info[request_user.id] = []
for tank in tanks_list:
rec = {
"value": tank["pk"],
"text": tank["fields"]["host"],
"shooting": {}
}
sh = active_shootings.filter(tank_id=tank["pk"])
if sh:
if len(sh) > 1:
log.warning("There are more than 1 active shooting "
"on the tank host")
shooting = sh[0]
default_data = \
self.get_default_data(shooting.scenario.scenario_path)
can_stop = request_user.id == shooting.user.id or \
self._salts_group in request_user.groups.all()
rec["shooting"] = {
"id": shooting.id,
"session": shooting.session_id,
"start": shooting.start,
"remained": remainedtime(shooting),
"scenario_id": shooting.scenario_id,
"username": shooting.user.username,
"default_data": default_data,
"custom_data": jsonstr2bin(str(shooting.custom_data)),
"can_stop": can_stop
}
self._actual_tanks_info[request_user.id].append(rec)
# records.append(json.dumps(rec))
return [
json.dumps(r) for r in self._actual_tanks_info[request_user.id]
]
def update(self, request, *args, **kwargs):
ex_data = {}
if request.META.get('HTTP_AUTHORIZATION'):
ex_data['token'] = \
request.META['HTTP_AUTHORIZATION'].replace('Token ', '')
self._add_ex_data(ex_data, request.data, 'force_run')
self._add_ex_data(ex_data, request.data, 'web_console_port')
try:
partial = kwargs.pop('partial', False)
instance = self.get_object()
serializer = self.get_serializer(instance,
data=request.data,
partial=partial)
serializer.is_valid(raise_exception=True)
self.perform_update(serializer, **ex_data)
return Response(serializer.data)
except ShootingHttpIssue, exc:
msg = "Shooting HTTP Issue: %s" % exc.message
log.warning(msg)
return Response(msg, status=exc.code, content_type='text/html')
def test_connection(target, port, hname=None):
try:
if hname:
req = "http://{hname}:5000/conn?target={target}&port={port}"
resp = requests.get(
req.format(hname=hname, target=target, port=port))
return float(resp.content)
else:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(1)
st = time.time()
s.connect((target, port))
s.close()
return time.time() - st
except Exception, exc:
if not hname:
hname = "localhost"
log.warning(
"The connection from %s to %s:%s is impossible due to %s. " %
(hname, target, port, exc))
return 0.0
def active_shootings(self):
shootings = Shooting.objects.filter(status='R')
invalid = []
for s in shootings:
current_time = time.time()
planned_finish = 0
if not s.start:
log.warning("The active shooting (id=%s) is invalid: "
"start time is unknown" % s.id)
invalid.append(s.id)
continue
if s.planned_duration:
planned_finish = s.start + s.planned_duration
else:
planned_finish = s.start + ScenarioRunView.MAX_TESTRUN_DURATION
if current_time > planned_finish:
log.warning("Likely the shooting (id=%s) is not running "
"at this time, but its status is 'Running'." %
s.id)
invalid.append(s.id)
return shootings.exclude(id__in=invalid)
def get_option_value(self,
scenario_path,
section,
option,
default_value=None):
config = ConfigParser()
ini_path = os.path.join(self.dir_path, scenario_path)
if not os.path.exists(ini_path):
log.warning("Config %s is not found." % ini_path)
return None
config.read(ini_path)
try:
return config.get(section, option)
except (NoSectionError, NoOptionError):
if default_value is None:
log.warning("Config {path}: "
"there is not '{option}' option "
"in the '{section}' section.".format(
path=ini_path, option=option, section=section))
return default_value
shooting.session_id)
try:
log.info("The test id=%s is stopped." % shooting.session_id)
self.free(shooting.tank.id)
if (resp and resp.get('current_stage') == 'poll') \
or (not resp and shooting.status != 'P'):
thread_data = {'shooting': shooting}
t = threading.Thread(name="Change Test Status",
target=self._change_test_status,
kwargs=thread_data)
t.start()
else:
log.info("Test id=%s won't be saved into DB "
"as it hasn't started yet." % shooting.session_id)
except Exception, exc:
log.warning("Exception when test "
"has been interrupted: %s" % exc)
def save_to_lock(self, tank_id, key, value):
if value is None:
return
lock_path = os.path.join(self.lock_dir_path, '%s.lock' % tank_id)
if os.path.exists(lock_path):
data = {}
with open(lock_path, 'rb') as f:
try:
data = pickle.load(f)
except EOFError:
data = {}
data[key] = value
with open(lock_path, 'wb') as f:
pickle.dump(data, f)
def get_test_status(self, request):
b_value = request_get_value(request, 'b')
scen_ids = []
if b_value:
jsonstr = bin2jsonstr(b_value)
scen_ids = json.loads(jsonstr)
scenarios = Scenario.objects.filter(
group_id__in=User.objects.get(id=request.user.id).groups.all(),
status='A')
if scen_ids:
scenarios = scenarios.filter(id__in=scen_ids)
sh = Shooting.objects.filter(scenario_id__in=scenarios.values('id'))
sh_max = sh.values('scenario_id').annotate(max_finish=Max('finish'))
tr = TestResult.objects.filter(
scenario_path__in=scenarios.values('scenario_path'))
tr_max = tr.values('scenario_path').annotate(
max_finish=Max('dt_finish'))
active_sh = self.active_shootings()
sort = request_get_value(request, "sort")
if sort:
order = request_get_value(request, "order")
reverse = ""
if not order:
order = "asc"
if order == "desc":
reverse = "-"
scenarios = scenarios.order_by("%sid" % reverse)
response_dict = {}
response_dict["total"] = len(scenarios)
offset = request_get_value(request, "offset")
limit = request_get_value(request, "limit")
if offset and limit:
offset = int(offset)
limit = int(limit)
scenarios = scenarios[offset:offset + limit]
results = []
# tanks = self.get_active_tanks(request.META["HTTP_HOST"])
tanks = self.get_active_tanks()
for s in scenarios:
values = {}
values["id"] = s.id
values["test_name"] = ini_manager.get_scenario_name(
s.scenario_path)
if not ini_manager.get_used_for_web(s.scenario_path):
log.warning("The {path} config is not used "
"via web.".format(path=s.scenario_path))
continue
values["default_data"] = self.get_default_data(s.scenario_path)
values["tank_host"] = {"id": "-1", "name": ""}
if "error" not in values["default_data"]:
try:
values["tank_host"] = self.select_tank_host(
tanks, active_sh, values["default_data"]["hostname"],
values["default_data"]["port"], s.scenario_path)
values["last"] = {}
shs = sh_max.filter(scenario_id=s.id)
trs = tr_max.filter(scenario_path=s.scenario_path)
if shs and trs:
trs = TestResult.objects.filter(
scenario_path=s.scenario_path,
dt_finish=trs[0]["max_finish"])
values["last"] = {
"tr_id": trs[0].id,
"finish": shs[0]["max_finish"]
}
except IniCtrlWarning, exc:
values["default_data"]["error"] = {
"name": exc.name,
"params": exc.params
}
results.append(values)
def __init__(self, name, params):
self.args = (name, params)
self.name = name
self.params = params
log.warning(IniCtrlWarning.messages[name].format(**params))
def is_valid_request(request, keys):
for k in keys:
if not request.POST.has_key(k):
log.warning("Request hasn't '%s' key." % k)
return False
return True
def check_changes(full_path):
entity = {"ts": None, "tool": []}
try:
config = ConfigParser.RawConfigParser()
config.read(full_path)
jmp = re.compile("jmeter")
ph = re.compile("phantom")
tool_sections = []
lt_tool = None
for sec in config.sections():
if ph.match(sec):
lt_tool = "phantom"
tool_sections.append(sec)
elif jmp.match(sec):
lt_tool = "jmeter"
tool_sections.append(sec)
if not lt_tool:
log.info("%s ini-file isn't config for tank test." % full_path)
log.warning(
"FUNC check_changes: %s ini-file isn't config for tank test." %
full_path)
return
file_name = full_path.replace("%s/" % LT_PATH, "")
try:
ts_record = TestSettings.objects.get(file_path=file_name)
except TestSettings.DoesNotExist:
ts_record = TestSettings(
file_path=file_name,
test_name="",
generator_id=localhost_generator_id(lt_tool),
ticket="",
version="")
ts_record.save()
entity["ts"] = ts_record
for sec in tool_sections:
(rps_value, target_host,
target_port) = get_config_values(config, sec, lt_tool)
tool_ent = {}
try:
target = Target.objects.get(host=target_host, port=target_port)
except Target.DoesNotExist:
target = Target(host=target_host, port=target_port)
target.save()
tool_ent["target"] = target
try:
rps = RPS.objects.get(test_settings_id=ts_record.id,
rps_name=sec)
except RPS.DoesNotExist:
rps = RPS(test_settings_id=ts_record.id,
rps_name=sec,
schedule=rps_value,
target_id=target.id)
rps.save()
else:
if not rps.target_id:
rps.target_id = target.id
if not rps.schedule == rps_value:
rps.schedule = rps_value
rps.save()
tool_ent["rps"] = rps
entity["tool"].append(tool_ent)
sec = "salts_report"
ts_record.test_name = config.get(sec, "test_name")
ts_record.ticket = config.get(sec, "ticket_url")
ts_record.version = config.get(sec, "version")
ts_record.save()
qs = RPS.objects.filter(test_settings_id=ts_record.id).exclude(
rps_name__in=tool_sections).delete()
return
except ConfigParser.NoOptionError as e:
log.warning("Config Parse Issue: %s. Ini-file: %s." % (e, full_path))
except ConfigParser.NoSectionError as e:
log.warning("Config Parse Issue: %s. Ini-file: %s." % (e, full_path))
except ConfigParser.MissingSectionHeaderError as e:
log.warning("Config Parse Issue: %s. Ini-file: %s." % (e, full_path))
except TankConfigError as e:
log.warning("Config Parse Issue: %s. Ini-file: %s." % (e, full_path))
for tool_ent in entity["tool"]:
if "target" in tool_ent:
tool_ent["target"].delete()
tool_ent["rps"].delete()
if entity["ts"]:
entity["ts"].delete()
