def _change_test_status(self, **kwargs): from salts.models import TestResult from datetime import timedelta shooting = kwargs.get('shooting') start_time = time.time() ctrl_c_delta = timedelta(seconds=TankManager.CTRL_C_INTERVAL) curr_time = time.time() while curr_time - start_time <= TankManager.WAIT_FOR_RESULT_SAVED: try: test_result = \ TestResult.objects.get(session_id=shooting.session_id) except TestResult.DoesNotExist: log.info("The test id=%s isn't " "been saved yet." % shooting.session_id) time.sleep(TankManager.POLL_INTERVAL) curr_time = time.time() continue if test_result.dt_finish - test_result.dt_start >= ctrl_c_delta: log.info("The test id=%s: " "test duration exceeds 3 minutes, " "the status remains Unknown." % shooting.session_id) return test_result.test_status = 'dbg' test_result.save() log.info("The test id=%s: " "test duration less than 3 minutes, " "the status is changed with Debug." % shooting.session_id) return log.warning("The test id=%s wasn't saved into DB." % shooting.session_id)
def ini_files(dir_path): ini = [] for root, dirs, files in os.walk(dir_path, topdown=False): for name in files: full_path = os.path.join(root, name) file_name, file_ext = os.path.splitext(full_path) if file_ext == '.ini': config = ConfigParser() try: config.read(full_path) except Error, exc: log.warning("Config %s is not valid: %s" % (full_path, exc)) else: try: test_name = config.get('salts_report', 'test_name') except (NoSectionError, NoOptionError): log.info("Config %s is not scenario: " "there is not 'test_name' option " "in the 'salts_report' section." % full_path) else: if test_name: ini.append(re.sub("%s/" % dir_path, '', full_path)) else: log.warning("Scenario %s contains " "empty 'test_name' option." % full_path)
def user_filter(request, results): scenario_id = request_get_value(request, "scid") scenario_path = request_get_value(request, "scpath") target = request_get_value(request, "trg") gen_type = request_get_value(request, "gt") generator = request_get_value(request, "gen") test_group = request_get_value(request, "tg") test_search = request_get_value(request, "ts") status = request_get_value(request, "st") task_id = request_get_value(request, "tid") spe = request_get_value(request, "spe") from_time = request_get_value(request, "from") to_time = request_get_value(request, "to") rps_value = request_get_value(request, "rps") if scenario_id: try: s = Scenario.objects.get(id=scenario_id) scenario_path = s.scenario_path except Scenario.DoesNotExist: log.warning("The scid=%s parameter from request does not exist.", scenario_id) if scenario_path: results = results.filter(scenario_path=scenario_path) if target: results = results.filter(target__contains=target) if test_group: results = results.filter(group=test_group) if test_search: results = results.filter(test_name__contains=test_search) if gen_type: results = results.filter(generator_types__name__contains=gen_type) if generator: results = results.filter(generator__contains=generator) if rps_value: results = results.filter(rps=rps_value) if status: status = status.split(",") for (i, item) in enumerate(status): if item == "unknown": status[i] = u"unk" if item == "debug": status[i] = u"dbg" results = results.filter(test_status__in=status) if task_id: results = results.filter(ticket_id=task_id) if spe: results = results.filter(user=spe) if from_time: from_time = from_time.replace("_", " ") results = results.filter(dt_finish__gte=from_time) if to_time: to_time = to_time.replace("_", " ") results = results.filter(dt_finish__lte=to_time) return results
def obtain_tank(self, tank_id, reqdata): try: reqdata["tank"] = Tank.objects.filter(id=tank_id) return None except Tank.DoesNotExist: resp = {'status': 'failed', 'message': "Given tank_id=%s isn't valid." \ % tank_id} log.warning(resp['message']) return HttpResponse(json.dumps(resp), content_type="application/json", status=404)
def obtain_scenario(self, scenario_id, reqdata): try: reqdata["scenario"] = Scenario.objects.filter(id=scenario_id) return None except Scenario.DoesNotExist: resp = {'status': 'failed', 'message': "Given scenario_id=%s isn't valid." \ % scenario_id} log.warning(resp['message']) return HttpResponse(json.dumps(resp), content_type="application/json", status=404)
def _wait_for_status(self, client, session_id): while True: resp = client.status(session_id) if "status" not in resp: log.warning("Response doesn't contain the 'status' field. " "Test with id=%s. Response: %s" % (session_id, resp)) return resp if resp["status"] == "running": time.sleep(TankManager.WAIT_INTERVAL) continue return resp
def get_queryset(self): host = self.request.query_params.get('host', None) if not host: return super(TankViewSet, self).get_queryset() try: info = socket.gethostbyname_ex(host) except: info = None if not info: log.warning("Invalid '%s' host name is given." % host) return Tank.objects.none() return Tank.objects.filter(host=info[0])
def check_auth(self, username, config): if 'api_user' in config['salts']: return None if username: config['salts']['api_user'] = username return None resp = {'status': 'failed', 'message': "User isn't authenticated " \ "or api_user option isn't provided."} log.warning(resp['message']) return HttpResponse(json.dumps(resp), content_type="application/json", status=401)
def get_used_for_web(self, scenario_path): config = ConfigParser() ini_path = os.path.join(self.dir_path, scenario_path) if not os.path.exists(ini_path): log.warning("Config %s is not found." % ini_path) return False config.read(ini_path) if "salts" not in config.sections(): return True if "used_for_web" not in config.options("salts"): return True bool_values = {"true": True, "false": False} str_value = config.get("salts", "used_for_web").lower() return bool_values.get(str_value, True)
def get_scenario_name(self, scenario_path): config = ConfigParser() ini_path = os.path.join(self.dir_path, scenario_path) test_name = '' if os.path.exists(ini_path): config.read(ini_path) try: test_name = config.get(IniCtrl.REPORT_SECTION, 'test_name') except (NoSectionError, NoOptionError): log.warning("Config %s is not scenario: " "there is not 'test_name' option " "in the 'salts_report' section." % ini_path) else: log.warning("Config %s is not found." % ini_path) return test_name
def run_test_api(request): if not is_valid_request(request, ["tsid"]): return HttpResponse("Request isn't valid.") sess = pickle.loads(request.session["test_run"]) tsid = request.POST["tsid"] if tsid in sess: msg = "FUNC run_test_api: client with id=%s exist already." % tsid log.warning(msg) return HttpResponse(msg) ts_record = TestSettings.objects.get(id=tsid) path = os.path.join(LT_PATH, ts_record.file_path) qs = TestSettings.objects.raw("SELECT g.id, g.host, g.port \ FROM salts_generator g \ JOIN salts_testsettings ts ON g.id = ts.generator_id WHERE ts.id = %s" % tsid) (gen_id, host, port) = (qs[0].id, qs[0].host, qs[0].port) client = TankClient(host, port, log) sess[tsid] = {} sess[tsid]["host"] = host sess[tsid]["port"] = port with open(path, "r") as ini_file: tr = TestRun(generator_id=gen_id, test_settings_id=tsid, status=TestRun.STATUS_RUNNING) tr.save() resp = client.run(ini_file.read(), "start", tr.id) if not resp: msg = "No server response when test tried to start." log.warning(msg) return HttpResponse(msg) else: sess[tsid]["trid"] = tr.id sess[tsid]["session"] = resp["session"] sess[tsid]["wait_status"] = "prepare" request.session["test_run"] = str(pickle.dumps(sess)) response_dict = {} response_dict.update({"ini_path": path}) response_dict.update({"wait_status": "prepare"}) response_dict.update({"session": resp["session"]}) return HttpResponse(json.dumps(response_dict), content_type="application/json")
def create(self, request, *args, **kwargs): ex_data = {} if request.META.get('HTTP_AUTHORIZATION'): ex_data['token'] = \ request.META['HTTP_AUTHORIZATION'].replace('Token ', '') self._add_ex_data(ex_data, request.data, 'force_run') try: serializer = self.get_serializer(data=request.data) serializer.is_valid(raise_exception=True) self.perform_create(serializer, **ex_data) headers = self.get_success_headers(serializer.data) return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers) except ShootingHttpIssue, exc: msg = "Shooting HTTP Issue: %s" % exc.message log.warning(msg) return Response(msg, status=exc.code, content_type='text/html')
def check_perm_start(self, config, scenario): username = config['salts']['api_user'] cursor = connection.cursor() cursor.execute(""" SELECT usr_gr.id FROM auth_user usr JOIN auth_user_groups usr_gr ON usr.id = usr_gr.user_id JOIN salts_scenario ss USING(group_id) WHERE ss.id = {scenario_id} and usr.username = '******' """.format(scenario_id=scenario.id, username=username)) if cursor.fetchone(): return None resp = {'status': 'failed', 'message': "Test %s disabled for '%s' user." \ % (scenario.scenario_path, config['salts']['api_user'])} log.warning(resp['message']) return HttpResponse(json.dumps(resp), content_type="application/json", status=403)
def check_perm_stop(self, username, shooting_id): cursor = connection.cursor() cursor.execute(""" SELECT sh.id FROM salts_shooting sh JOIN auth_user_groups usr_gr USING(user_id) JOIN auth_user_groups usr_gr_req ON usr_gr_req.group_id=usr_gr.group_id JOIN auth_user usr_req ON usr_req.id=usr_gr_req.user_id WHERE sh.id={shooting_id} AND usr_req.username='******' """.format(shooting_id=shooting_id, req_username=username)) if cursor.fetchone(): return None resp = {'status': 'failed', 'message': "Shooting cannot be stopped by '%s' user." \ % username} log.warning(resp['message']) return HttpResponse(json.dumps(resp), content_type="application/json", status=403)
def adapt_tanks_list(self, tanks_list, active_shootings, request_user): if request_user.id in self._actual_tanks_info: for record in self._actual_tanks_info[request_user.id]: sh = active_shootings.filter(id=record["shooting"]["id"]) if sh: record["shooting"]["remained"] = remainedtime(sh[0]) else: self._actual_tanks_info[request_user.id]["shooting"] = {} else: self._actual_tanks_info[request_user.id] = [] for tank in tanks_list: rec = { "value": tank["pk"], "text": tank["fields"]["host"], "shooting": {} } sh = active_shootings.filter(tank_id=tank["pk"]) if sh: if len(sh) > 1: log.warning("There are more than 1 active shooting " "on the tank host") shooting = sh[0] default_data = \ self.get_default_data(shooting.scenario.scenario_path) can_stop = request_user.id == shooting.user.id or \ self._salts_group in request_user.groups.all() rec["shooting"] = { "id": shooting.id, "session": shooting.session_id, "start": shooting.start, "remained": remainedtime(shooting), "scenario_id": shooting.scenario_id, "username": shooting.user.username, "default_data": default_data, "custom_data": jsonstr2bin(str(shooting.custom_data)), "can_stop": can_stop } self._actual_tanks_info[request_user.id].append(rec) # records.append(json.dumps(rec)) return [ json.dumps(r) for r in self._actual_tanks_info[request_user.id] ]
def update(self, request, *args, **kwargs): ex_data = {} if request.META.get('HTTP_AUTHORIZATION'): ex_data['token'] = \ request.META['HTTP_AUTHORIZATION'].replace('Token ', '') self._add_ex_data(ex_data, request.data, 'force_run') self._add_ex_data(ex_data, request.data, 'web_console_port') try: partial = kwargs.pop('partial', False) instance = self.get_object() serializer = self.get_serializer(instance, data=request.data, partial=partial) serializer.is_valid(raise_exception=True) self.perform_update(serializer, **ex_data) return Response(serializer.data) except ShootingHttpIssue, exc: msg = "Shooting HTTP Issue: %s" % exc.message log.warning(msg) return Response(msg, status=exc.code, content_type='text/html')
def test_connection(target, port, hname=None): try: if hname: req = "http://{hname}:5000/conn?target={target}&port={port}" resp = requests.get( req.format(hname=hname, target=target, port=port)) return float(resp.content) else: s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.settimeout(1) st = time.time() s.connect((target, port)) s.close() return time.time() - st except Exception, exc: if not hname: hname = "localhost" log.warning( "The connection from %s to %s:%s is impossible due to %s. " % (hname, target, port, exc)) return 0.0
def active_shootings(self): shootings = Shooting.objects.filter(status='R') invalid = [] for s in shootings: current_time = time.time() planned_finish = 0 if not s.start: log.warning("The active shooting (id=%s) is invalid: " "start time is unknown" % s.id) invalid.append(s.id) continue if s.planned_duration: planned_finish = s.start + s.planned_duration else: planned_finish = s.start + ScenarioRunView.MAX_TESTRUN_DURATION if current_time > planned_finish: log.warning("Likely the shooting (id=%s) is not running " "at this time, but its status is 'Running'." % s.id) invalid.append(s.id) return shootings.exclude(id__in=invalid)
def get_option_value(self, scenario_path, section, option, default_value=None): config = ConfigParser() ini_path = os.path.join(self.dir_path, scenario_path) if not os.path.exists(ini_path): log.warning("Config %s is not found." % ini_path) return None config.read(ini_path) try: return config.get(section, option) except (NoSectionError, NoOptionError): if default_value is None: log.warning("Config {path}: " "there is not '{option}' option " "in the '{section}' section.".format( path=ini_path, option=option, section=section)) return default_value
shooting.session_id) try: log.info("The test id=%s is stopped." % shooting.session_id) self.free(shooting.tank.id) if (resp and resp.get('current_stage') == 'poll') \ or (not resp and shooting.status != 'P'): thread_data = {'shooting': shooting} t = threading.Thread(name="Change Test Status", target=self._change_test_status, kwargs=thread_data) t.start() else: log.info("Test id=%s won't be saved into DB " "as it hasn't started yet." % shooting.session_id) except Exception, exc: log.warning("Exception when test " "has been interrupted: %s" % exc) def save_to_lock(self, tank_id, key, value): if value is None: return lock_path = os.path.join(self.lock_dir_path, '%s.lock' % tank_id) if os.path.exists(lock_path): data = {} with open(lock_path, 'rb') as f: try: data = pickle.load(f) except EOFError: data = {} data[key] = value with open(lock_path, 'wb') as f: pickle.dump(data, f)
def get_test_status(self, request): b_value = request_get_value(request, 'b') scen_ids = [] if b_value: jsonstr = bin2jsonstr(b_value) scen_ids = json.loads(jsonstr) scenarios = Scenario.objects.filter( group_id__in=User.objects.get(id=request.user.id).groups.all(), status='A') if scen_ids: scenarios = scenarios.filter(id__in=scen_ids) sh = Shooting.objects.filter(scenario_id__in=scenarios.values('id')) sh_max = sh.values('scenario_id').annotate(max_finish=Max('finish')) tr = TestResult.objects.filter( scenario_path__in=scenarios.values('scenario_path')) tr_max = tr.values('scenario_path').annotate( max_finish=Max('dt_finish')) active_sh = self.active_shootings() sort = request_get_value(request, "sort") if sort: order = request_get_value(request, "order") reverse = "" if not order: order = "asc" if order == "desc": reverse = "-" scenarios = scenarios.order_by("%sid" % reverse) response_dict = {} response_dict["total"] = len(scenarios) offset = request_get_value(request, "offset") limit = request_get_value(request, "limit") if offset and limit: offset = int(offset) limit = int(limit) scenarios = scenarios[offset:offset + limit] results = [] # tanks = self.get_active_tanks(request.META["HTTP_HOST"]) tanks = self.get_active_tanks() for s in scenarios: values = {} values["id"] = s.id values["test_name"] = ini_manager.get_scenario_name( s.scenario_path) if not ini_manager.get_used_for_web(s.scenario_path): log.warning("The {path} config is not used " "via web.".format(path=s.scenario_path)) continue values["default_data"] = self.get_default_data(s.scenario_path) values["tank_host"] = {"id": "-1", "name": ""} if "error" not in values["default_data"]: try: values["tank_host"] = self.select_tank_host( tanks, active_sh, values["default_data"]["hostname"], values["default_data"]["port"], s.scenario_path) values["last"] = {} shs = sh_max.filter(scenario_id=s.id) trs = tr_max.filter(scenario_path=s.scenario_path) if shs and trs: trs = TestResult.objects.filter( scenario_path=s.scenario_path, dt_finish=trs[0]["max_finish"]) values["last"] = { "tr_id": trs[0].id, "finish": shs[0]["max_finish"] } except IniCtrlWarning, exc: values["default_data"]["error"] = { "name": exc.name, "params": exc.params } results.append(values)
def __init__(self, name, params): self.args = (name, params) self.name = name self.params = params log.warning(IniCtrlWarning.messages[name].format(**params))
def is_valid_request(request, keys): for k in keys: if not request.POST.has_key(k): log.warning("Request hasn't '%s' key." % k) return False return True
def check_changes(full_path): entity = {"ts": None, "tool": []} try: config = ConfigParser.RawConfigParser() config.read(full_path) jmp = re.compile("jmeter") ph = re.compile("phantom") tool_sections = [] lt_tool = None for sec in config.sections(): if ph.match(sec): lt_tool = "phantom" tool_sections.append(sec) elif jmp.match(sec): lt_tool = "jmeter" tool_sections.append(sec) if not lt_tool: log.info("%s ini-file isn't config for tank test." % full_path) log.warning( "FUNC check_changes: %s ini-file isn't config for tank test." % full_path) return file_name = full_path.replace("%s/" % LT_PATH, "") try: ts_record = TestSettings.objects.get(file_path=file_name) except TestSettings.DoesNotExist: ts_record = TestSettings( file_path=file_name, test_name="", generator_id=localhost_generator_id(lt_tool), ticket="", version="") ts_record.save() entity["ts"] = ts_record for sec in tool_sections: (rps_value, target_host, target_port) = get_config_values(config, sec, lt_tool) tool_ent = {} try: target = Target.objects.get(host=target_host, port=target_port) except Target.DoesNotExist: target = Target(host=target_host, port=target_port) target.save() tool_ent["target"] = target try: rps = RPS.objects.get(test_settings_id=ts_record.id, rps_name=sec) except RPS.DoesNotExist: rps = RPS(test_settings_id=ts_record.id, rps_name=sec, schedule=rps_value, target_id=target.id) rps.save() else: if not rps.target_id: rps.target_id = target.id if not rps.schedule == rps_value: rps.schedule = rps_value rps.save() tool_ent["rps"] = rps entity["tool"].append(tool_ent) sec = "salts_report" ts_record.test_name = config.get(sec, "test_name") ts_record.ticket = config.get(sec, "ticket_url") ts_record.version = config.get(sec, "version") ts_record.save() qs = RPS.objects.filter(test_settings_id=ts_record.id).exclude( rps_name__in=tool_sections).delete() return except ConfigParser.NoOptionError as e: log.warning("Config Parse Issue: %s. Ini-file: %s." % (e, full_path)) except ConfigParser.NoSectionError as e: log.warning("Config Parse Issue: %s. Ini-file: %s." % (e, full_path)) except ConfigParser.MissingSectionHeaderError as e: log.warning("Config Parse Issue: %s. Ini-file: %s." % (e, full_path)) except TankConfigError as e: log.warning("Config Parse Issue: %s. Ini-file: %s." % (e, full_path)) for tool_ent in entity["tool"]: if "target" in tool_ent: tool_ent["target"].delete() tool_ent["rps"].delete() if entity["ts"]: entity["ts"].delete()