def book(self, tank_id): lock_path = os.path.join(self.lock_dir_path, '%s.lock' % tank_id) if os.path.exists(lock_path): return False open(lock_path, 'w').close() log.info("Tank with id=%s was busy." % tank_id) return True
def create(self, validated_data): log.info("ShootingSerializer.create. " "validated_data: %s" % validated_data) scenario = validated_data.get('scenario') tank = validated_data.get('tank') if not self._get_force_run(validated_data.get('force_run')): self.check_permission(validated_data.get('token'), scenario) if not tank_manager.book(tank.id): raise ShootingHttpIssue(status.HTTP_403_FORBIDDEN, "Tank is busy on host %s" % tank.host) token = Token.objects.get(key=validated_data.get('token')) alt_name = validated_data.get('alt_name') if not alt_name: alt_name = token.user.username sh_data = { 'scenario_id': scenario.id, 'tank_id': tank.id, 'user_id': token.user.id, 'status': validated_data.get('status'), 'session_id': validated_data.get('session_id'), 'ticket_id': validated_data.get('ticket_id'), 'alt_name': alt_name } shooting = Shooting.objects.create(**sh_data) tank_manager.save_to_lock(tank.id, 'session_id', validated_data.get('session_id')) return shooting
def show_test_settings(request): if request.method == "GET": for file_path in ini_files(): check_changes(file_path) return TestSettingsList.as_view()(request) if request.method == "POST": if "cancel-button" in request.POST: return HttpResponseRedirect("/tests/") log.info("Request.POST: %s" % request.POST) ts_record = TestSettings.objects.get(id=request.POST["settings"]) config_path = os.path.join(LT_PATH, ts_record.file_path) log.info("ts_record: %s" % ts_record) ts_record.test_name = request.POST["test_name"] ts_record.ticket = request.POST["ticket"] ts_record.version = request.POST["version"] ts_record.generator_id = request.POST["generator"] ts_record.save() sync_config(config_path, test_settings=ts_record) rpsid = request.POST["rpsid"].split(",") form_id = 0 for id_rps in rpsid: rps_record = RPS.objects.get(id=id_rps) rps_record.rps_name = request.POST["form-%d-rps_name" % form_id] rps_record.schedule = request.POST["form-%d-schedule" % form_id] rps_record.target_id = request.POST["form-%d-target" % form_id] rps_record.save() sync_config(config_path, rps=rps_record) form_id += 1 return HttpResponseRedirect("/tests/")
def sync_config(file_path, *args, **kwargs): config = UnicodeConfigParser() config.readfp(codecs.open(file_path, "r", "utf-8")) is_changed = False if "test_settings" in kwargs: ts_record = kwargs["test_settings"] sp_sec = "salts_report" config.set(sp_sec, "test_name", ts_record.test_name) log.info("INFO: %s" % ts_record.test_name) config.set(sp_sec, "ticket_url", ts_record.ticket) config.set(sp_sec, "version", ts_record.version) is_changed = True if "rps" in kwargs: ph = re.compile("phantom") jmp = re.compile("jmeter") rps_record = kwargs["rps"] if ph.match(rps_record.rps_name): config.set(rps_record.rps_name, "rps_schedule", rps_record.schedule) is_changed = True elif jmp.match(rps_record.rps_name): (rps, rampup, testlen, rampdown) = re.findall("\d+", rps_record.schedule) config.set(rps_record.rps_name, "rps1", rps) config.set(rps_record.rps_name, "rps2", rps) config.set(rps_record.rps_name, "rampup", rampup) config.set(rps_record.rps_name, "testlen", testlen) config.set(rps_record.rps_name, "rampdown", rampdown) is_changed = True if is_changed: config.write(codecs.open(file_path, "wb", "utf-8"))
def free(self, tank_id): lock_path = os.path.join(self.lock_dir_path, '%s.lock' % tank_id) if not os.path.exists(lock_path): return False os.unlink(lock_path) log.info("Tank with id=%s became free." % tank_id) return True
def edit_test_parameters(request, settings_id): ts_record = TestSettings.objects.get(id=settings_id) settings_form = SettingsEditForm(instance=ts_record) rps_record = RPS.objects.filter(test_settings_id=settings_id) RpsFormSet = formset_factory(RPSEditForm, extra=0) # log.info("rps_record: %s" % rps_record) data = [] rps_id = [] for record in rps_record: rps_id.append(str(record.id)) log.info("RPS Edit Form: %s" % record.rps_name) data.append({ "target": record.target, "rps_name": record.rps_name, "schedule": record.schedule }) # rps_form = RPSEditForm(instance=record) formset = RpsFormSet(initial=data) context = {} context.update(csrf(request)) context.update({ "settings_form": settings_form, "rps_form": formset, "settings": settings_id, "rpsid": ",".join(rps_id) }) return render_to_response("testsettings_edit.html", context)
def ini_files(dir_path): ini = [] for root, dirs, files in os.walk(dir_path, topdown=False): for name in files: full_path = os.path.join(root, name) file_name, file_ext = os.path.splitext(full_path) if file_ext == '.ini': config = ConfigParser() try: config.read(full_path) except Error, exc: log.warning("Config %s is not valid: %s" % (full_path, exc)) else: try: test_name = config.get('salts_report', 'test_name') except (NoSectionError, NoOptionError): log.info("Config %s is not scenario: " "there is not 'test_name' option " "in the 'salts_report' section." % full_path) else: if test_name: ini.append(re.sub("%s/" % dir_path, '', full_path)) else: log.warning("Scenario %s contains " "empty 'test_name' option." % full_path)
def get_tank_status(request): log.info("get_tank_status: request.GET: %s" % request.GET) tank_id = request.GET.get('tank_id') if tank_id: tanks = Tank.objects.filter(id=tank_id) else: tanks = Tank.objects.all() results = [] for t in tanks: shooting = t.shooting_set.exclude( start__isnull=True).order_by('-start').first() if not shooting: continue username = '' if shooting.user: if shooting.user.username == 'ltbot': username = shooting.alt_name # временно, для консольных # тестов, чтобы обойти # аутентификацию else: username = shooting.user.username scenario_path = shooting.scenario.scenario_path values = {'id': t.id, 'host': t.host, 'username': username, 'gitlab_url': '%s%s' % (LT_GITLAB, shooting.scenario.scenario_path), 'scenario_name': \ ini_manager.get_scenario_name(scenario_path), 'status': shooting.status, 'countdown': remainedtime(shooting), 'shooting_id': shooting.id, 'ticket_id': shooting.ticket_id} port = tank_manager.read_from_lock(t.id, 'web_console_port') if port: values['webconsole'] = "%s:%s" % (t.host, port) if shooting.ticket_id: values['ticket_url'] = '%s%s' % (LT_JIRA, shooting.ticket_id) if shooting.status in ['F', 'I']: tr = TestResult.objects.filter(session_id=shooting.session_id) if tr: values['test_result'] = tr[0].id results.append(values) sort_param = request_get_value(request, 'sort') if sort_param: order = request_get_value(request, 'order') if not order: order = 'asc' reverse = order == 'desc' results = sorted(results, key=itemgetter('id'), reverse=reverse) response_dict = {} response_dict['total'] = len(results) response_dict['rows'] = results response = HttpResponse(json.dumps(response_dict), content_type='application/json') add_version(response) return response
def __init__(self, root): self.dir_path = root try: g = Group.objects.get(name=IniCtrl.DEFAULT_GROUP) except Group.DoesNotExist: g = Group(name=IniCtrl.DEFAULT_GROUP) g.save() log.info("Group '%s' has been added." % IniCtrl.DEFAULT_GROUP) self.default_group_id = g.id
def create(self, validated_data): log.info("TestResult: validated_data: %s" % validated_data) gt_data = validated_data.pop("generator_types") test_result = TestResult.objects.create(**validated_data) test_result.save() for gt in gt_data: for k in gt: gen_type = GeneratorType.objects.get(name=gt[k]) test_result.generator_types.add(gen_type) return test_result
def interrupt(self, shooting): resp = None if re.match('\d+_0+', shooting.session_id): try: client = TankClient(shooting.tank.host, shooting.tank.port) resp = client.status(shooting.session_id) client.stop(shooting.session_id) except Exception, exc: log.info("Exception when test " "has been interrupted: %s" % exc)
def _check_for_running(self, client): while True: resp = client.status() running = False for sess_id in resp: if resp[sess_id]["status"] == "running": log.info("Test can't start because other " "test with id=%s is running now." % sess_id) running = True break if not running: break time.sleep(TankManager.POLL_INTERVAL)
def shoot(self, **kwargs): custom_data = kwargs.get('custom_data') scenario = kwargs.get('scenario') tank = kwargs.get('tank') client = TankClient(tank.host, tank.port) config = CustomConfig(os.path.join(LT_PATH, scenario.scenario_path)) config.mergejson(custom_data) resp = None resp = client.run(config.textcontent(), 'start') session_id = resp['session'] log.info("Test with id=%s started." % session_id) self._wait_for_completed(client, session_id, tank.id, False) client.resume(session_id) response = self._wait_for_completed(client, session_id, tank.id, True) log.info("Test with id=%s stopped." % session_id) return response
def update(self, instance, validated_data): log.info("Shooting. Update: validated_data: %s" % validated_data) if not self._get_force_run(validated_data.get('force_run')): self.check_permission(validated_data.get('token'), instance.scenario) tank_manager.save_to_lock(instance.tank.id, 'web_console_port', validated_data.get('web_console_port')) fields = [] for k in validated_data: if k in self.updated_fields: setattr(instance, k, validated_data.get(k, getattr(instance, k))) fields.append(k) instance.save(update_fields=fields) return instance
def _scenario_id_from_ini(self, scenario_path): config = ConfigParser() config.read(os.path.join(self.dir_path, scenario_path)) if not config.has_section(IniCtrl.SALTS_SECTION): return 0 if config.has_option(IniCtrl.SALTS_SECTION, IniCtrl.SCENARIO_ID_OPTION): return int( config.get(IniCtrl.SALTS_SECTION, IniCtrl.SCENARIO_ID_OPTION)) if config.has_option(IniCtrl.SALTS_SECTION, 'test_ini_id'): log.info("Scenario %s: " "'test_ini_id' option is deprecated. " "It won't be supported in future versions. " "Please use 'scenario_id' option " "instead of it." % os.path.join(self.dir_path, scenario_path)) return int(config.get(IniCtrl.SALTS_SECTION, 'test_ini_id')) return 0
def _change_test_status(self, **kwargs): from salts.models import TestResult from datetime import timedelta shooting = kwargs.get('shooting') start_time = time.time() ctrl_c_delta = timedelta(seconds=TankManager.CTRL_C_INTERVAL) curr_time = time.time() while curr_time - start_time <= TankManager.WAIT_FOR_RESULT_SAVED: try: test_result = \ TestResult.objects.get(session_id=shooting.session_id) except TestResult.DoesNotExist: log.info("The test id=%s isn't " "been saved yet." % shooting.session_id) time.sleep(TankManager.POLL_INTERVAL) curr_time = time.time() continue if test_result.dt_finish - test_result.dt_start >= ctrl_c_delta: log.info("The test id=%s: " "test duration exceeds 3 minutes, " "the status remains Unknown." % shooting.session_id) return test_result.test_status = 'dbg' test_result.save() log.info("The test id=%s: " "test duration less than 3 minutes, " "the status is changed with Debug." % shooting.session_id) return log.warning("The test id=%s wasn't saved into DB." % shooting.session_id)
class TankManager(object): CTRL_C_INTERVAL = 180 # seconds (TESTING-2586) POLL_INTERVAL = 5 # seconds WAIT_INTERVAL = 1 # seconds WAIT_FOR_RESULT_SAVED = 60 # seconds def __init__(self): test_lock_dir_path(LOCK_PATH) self.lock_dir_path = LOCK_PATH def book(self, tank_id): lock_path = os.path.join(self.lock_dir_path, '%s.lock' % tank_id) if os.path.exists(lock_path): return False open(lock_path, 'w').close() log.info("Tank with id=%s was busy." % tank_id) return True def free(self, tank_id): lock_path = os.path.join(self.lock_dir_path, '%s.lock' % tank_id) if not os.path.exists(lock_path): return False os.unlink(lock_path) log.info("Tank with id=%s became free." % tank_id) return True def _check_for_running(self, client): while True: resp = client.status() running = False for sess_id in resp: if resp[sess_id]["status"] == "running": log.info("Test can't start because other " "test with id=%s is running now." % sess_id) running = True break if not running: break time.sleep(TankManager.POLL_INTERVAL) def _wait_for_completed(self, client, session_id, tank_id, expected_retcode): def format_resp(resp): failures = resp.get('failures') if failures: for fail in failures: fail['reason'] = fail['reason'].split('\n') return json.dumps(resp, indent=4) while True: resp = client.status(session_id) if "stage_completed" in resp: status = resp["status"] completed = stg_completed_to_bool(resp["stage_completed"]) if expected_retcode: if resp["retcode"] is None: completed = False if completed: log.debug("Test %s. Response: %s" % (status, format_resp(resp))) return resp log.debug("Test %s. Response: %s" % (status, format_resp(resp))) else: log.debug("Response: %s" % format_resp(resp)) time.sleep(TankManager.WAIT_INTERVAL) def _wait_for_status(self, client, session_id): while True: resp = client.status(session_id) if "status" not in resp: log.warning("Response doesn't contain the 'status' field. " "Test with id=%s. Response: %s" % (session_id, resp)) return resp if resp["status"] == "running": time.sleep(TankManager.WAIT_INTERVAL) continue return resp def shoot(self, **kwargs): custom_data = kwargs.get('custom_data') scenario = kwargs.get('scenario') tank = kwargs.get('tank') client = TankClient(tank.host, tank.port) config = CustomConfig(os.path.join(LT_PATH, scenario.scenario_path)) config.mergejson(custom_data) resp = None resp = client.run(config.textcontent(), 'start') session_id = resp['session'] log.info("Test with id=%s started." % session_id) self._wait_for_completed(client, session_id, tank.id, False) client.resume(session_id) response = self._wait_for_completed(client, session_id, tank.id, True) log.info("Test with id=%s stopped." % session_id) return response def shootmq(self, tank, scenario, custom_data): tank_fields = tank[0]["fields"] tank_id = tank[0]["pk"] scenario_fields = scenario[0]["fields"] try: client = TankClient(tank_fields["host"], tank_fields["port"]) except TankClientError, exc: resp = { "status": "failed", "failures": [{ "reason": str(exc), "stage": "prepare" }] } return resp config = CustomConfig( os.path.join(LT_PATH, scenario_fields["scenario_path"])) config.mergejson(custom_data) resp = None resp = client.run(config.textcontent(), "start") session_id = resp["session"] log.info("Test with id=%s started." % session_id) self._wait_for_completed(client, session_id, tank_id, False) client.resume(session_id) self._wait_for_completed(client, session_id, tank_id, True) resp = self._wait_for_status(client, session_id) log.info("Test with id=%s stopped." % session_id) return resp
def check_changes(full_path): entity = {"ts": None, "tool": []} try: config = ConfigParser.RawConfigParser() config.read(full_path) jmp = re.compile("jmeter") ph = re.compile("phantom") tool_sections = [] lt_tool = None for sec in config.sections(): if ph.match(sec): lt_tool = "phantom" tool_sections.append(sec) elif jmp.match(sec): lt_tool = "jmeter" tool_sections.append(sec) if not lt_tool: log.info("%s ini-file isn't config for tank test." % full_path) log.warning( "FUNC check_changes: %s ini-file isn't config for tank test." % full_path) return file_name = full_path.replace("%s/" % LT_PATH, "") try: ts_record = TestSettings.objects.get(file_path=file_name) except TestSettings.DoesNotExist: ts_record = TestSettings( file_path=file_name, test_name="", generator_id=localhost_generator_id(lt_tool), ticket="", version="") ts_record.save() entity["ts"] = ts_record for sec in tool_sections: (rps_value, target_host, target_port) = get_config_values(config, sec, lt_tool) tool_ent = {} try: target = Target.objects.get(host=target_host, port=target_port) except Target.DoesNotExist: target = Target(host=target_host, port=target_port) target.save() tool_ent["target"] = target try: rps = RPS.objects.get(test_settings_id=ts_record.id, rps_name=sec) except RPS.DoesNotExist: rps = RPS(test_settings_id=ts_record.id, rps_name=sec, schedule=rps_value, target_id=target.id) rps.save() else: if not rps.target_id: rps.target_id = target.id if not rps.schedule == rps_value: rps.schedule = rps_value rps.save() tool_ent["rps"] = rps entity["tool"].append(tool_ent) sec = "salts_report" ts_record.test_name = config.get(sec, "test_name") ts_record.ticket = config.get(sec, "ticket_url") ts_record.version = config.get(sec, "version") ts_record.save() qs = RPS.objects.filter(test_settings_id=ts_record.id).exclude( rps_name__in=tool_sections).delete() return except ConfigParser.NoOptionError as e: log.warning("Config Parse Issue: %s. Ini-file: %s." % (e, full_path)) except ConfigParser.NoSectionError as e: log.warning("Config Parse Issue: %s. Ini-file: %s." % (e, full_path)) except ConfigParser.MissingSectionHeaderError as e: log.warning("Config Parse Issue: %s. Ini-file: %s." % (e, full_path)) except TankConfigError as e: log.warning("Config Parse Issue: %s. Ini-file: %s." % (e, full_path)) for tool_ent in entity["tool"]: if "target" in tool_ent: tool_ent["target"].delete() tool_ent["rps"].delete() if entity["ts"]: entity["ts"].delete()
"the status is changed with Debug." % shooting.session_id) return log.warning("The test id=%s wasn't saved into DB." % shooting.session_id) def interrupt(self, shooting): resp = None if re.match('\d+_0+', shooting.session_id): try: client = TankClient(shooting.tank.host, shooting.tank.port) resp = client.status(shooting.session_id) client.stop(shooting.session_id) except Exception, exc: log.info("Exception when test " "has been interrupted: %s" % exc) log.info("TankManager.interrupt. Shooting.session_id: %s" % shooting.session_id) try: log.info("The test id=%s is stopped." % shooting.session_id) self.free(shooting.tank.id) if (resp and resp.get('current_stage') == 'poll') \ or (not resp and shooting.status != 'P'): thread_data = {'shooting': shooting} t = threading.Thread(name="Change Test Status", target=self._change_test_status, kwargs=thread_data) t.start() else: log.info("Test id=%s won't be saved into DB " "as it hasn't started yet." % shooting.session_id) except Exception, exc: log.warning("Exception when test "
def gitsync(request): log.info("gitsync calling") return HttpResponse(status=200)
def start_shooting(self, scenario_id, tank_id, custom_data, request): username = request.user.username reqdata = {} err = self.obtain_scenario(scenario_id, reqdata) if err: return err err = self.obtain_tank(tank_id, reqdata) if err: return err json_str = '{}' if custom_data: json_str = bin2jsonstr(custom_data) config = json.loads(json_str) json_str = json.dumps(config) if "salts" not in config: config["salts"] = {} server_addr = "http://{addr}".format(addr=request.get_host()) config["salts"]["api_host"] = server_addr config["salts"]["api_url"] = server_addr + "/api2" err = self.check_auth(username, config) if err: return err err = self.check_perm_start(config, reqdata["scenario"][0]) if err: return err if 'api_key' not in config['salts']: user = User.objects.get(username=config['salts']['api_user']) tokens = Token.objects.filter(user_id=user.id) config['salts']['api_key'] = tokens[0].key reqdata['custom_data'] = json.dumps(config) task_id = shoot.delay( json.loads(serializers.serialize("json", reqdata["tank"])), json.loads(serializers.serialize("json", reqdata["scenario"])), reqdata["custom_data"]) custom_saved = False session_id = None shooting = None curr_time = start_time = time.time() while curr_time - start_time < ShooterView.MAX_WAIT_FOR_SHOOTING_START: log.info("Wait for shooting start.") time.sleep(1) if not session_id: session_id = tank_manager.read_from_lock(tank_id, 'session_id') if session_id: shooting = Shooting.objects.get(session_id=session_id) custom_saved = self.save_custom_data(shooting, json_str, custom_saved) if shooting and shooting.status == 'R': resp = { 'status': 'success', 'id': shooting.id, 'session': shooting.session_id, 'start': shooting.start, 'duration': shooting.planned_duration, 'custom_data': jsonstr2bin(shooting.custom_data) } return HttpResponse(json.dumps(resp), content_type="application/json") err_resp = self._check_for_error(task_id) if err_resp: self.save_custom_data(shooting, reqdata['custom_data'], custom_saved) return err_resp curr_time = time.time() if shooting: tank_manager.interrupt(shooting) return HttpResponse(status=408)
def log_message(msg): log.info(msg)