def output_mismatch_case(anchor, test):
result = '<html>\n<head>\n'
for scenario in gp.scenario:
anchor_dir = gp.generate_dir_path(anchor, scenario)
test_dir = gp.generate_dir_path(test, scenario)
header = '<hr>\n'
header += '<h2>\nComparing ' + scenario + ' result:\n</h2>\n'
result += header
mismatch_packet = MisMatchPacket()
crash_packet = MisMatchPacket()
if os.path.isdir(test_dir):
room_list = os.listdir(test_dir)
for room in room_list:
test_room_dir = gp.generate_dir_path(test_dir, room)
anchor_room_dir = gp.generate_dir_path(anchor_dir, room)
if os.path.isdir(test_room_dir) and os.path.isdir(
anchor_room_dir):
is_crash = False
doc_list = os.listdir(test_room_dir)
for doc in doc_list:
if re.search('crash.txt', doc):
is_crash = True
doc_list = os.listdir(anchor_room_dir)
for doc in doc_list:
if re.search('crash.txt', doc):
is_crash = True
if is_crash:
crash_packet = add_one_case(crash_packet, room)
else:
mismatch_packet = add_one_case(mismatch_packet, room)
crash_packet = finish_one_case(crash_packet)
mismatch_packet = finish_one_case(mismatch_packet)
if crash_packet.number_ == 0 and mismatch_packet.number_ == 0:
result += '<p>All Match!!!</p>\n'
else:
if mismatch_packet.number_ != 0:
result += '<p>Mismatch Case:</p>\n'
result += mismatch_packet.case_
if crash_packet.number_ != 0:
result += '<p>Crash Case:</p>\n'
result += crash_packet.case_
result += '</head>\n</html>\n'
return result
class Client:
def __init__(self, case, idx, uid, total_client):
orig_sequence = gp.sequences[case[7][idx]]
self.case_ = case[0]
self.duration_ = case[2]
self.resolution_ = case[3][idx]
self.rate_ = case[4][idx]
self.fps_ = case[5][idx]
self.network_ = case[6][idx]
self.sequence_dir_ = gp.generate_dir_path(gp.sequence_dir,
orig_sequence)
self.room_ = int(self.case_.split(gp.folder_join)[-1], 16)
self.uid_ = uid
self.capacity_ = int(self.resolution_.split('x')[0]) * int(self.resolution_.split('x')[1]) \
* self.fps_ * (2 + total_client - 1) / 2
tmp = orig_sequence.split('_')
self.anchor_seq_ = tmp[0] + '_' + tmp[1] + '_' + tmp[2] + '_' + self.resolution_ + 'p' + str(self.fps_) \
+ '_' + tmp[4] + '.yuv'
self.config_ = 'Rate' + str(self.rate_) + gp.folder_join + 'Net' + str(self.network_) + gp.folder_join \
+ 'FPS' + str(self.fps_) + gp.folder_join + 'Res' + self.resolution_ + gp.folder_join \
+ 'Seq' + tmp[1]
key = case[7][idx]
if key in gp.seq_candidates:
if self.anchor_seq_ not in gp.seq_candidates[key]:
gp.seq_candidates[key].append(self.anchor_seq_)
else:
gp.seq_candidates[key] = [self.anchor_seq_]
def copy_corresponding_log(case_name, scenario):
case_dir = gp.generate_dir_path(gp.problem_case_dir, scenario, case_name)
if os.path.exists(case_dir):
return
cur_case = find_corresponding_case_path(case_name, scenario,
gp.cur_log_dir)
dest_dir = gp.generate_dir_path(gp.problem_case_dir, scenario, case_name,
'cur')
gp.create_dir(dest_dir)
os.system('cp -rf ' + cur_case + '* ' + dest_dir)
ref_case = find_corresponding_case_path(case_name, scenario,
gp.ref_log_dir)
dest_dir = gp.generate_dir_path(gp.problem_case_dir, scenario, case_name,
'ref')
gp.create_dir(dest_dir)
os.system('cp -rf ' + ref_case + '* ' + dest_dir)
def scale_yuv(*args):
seq_idx = args[0]
gp.print_log(gp.LogLevel.Normal,
'\tScaling ' + gp.sequences[seq_idx] + '...')
ori_sequence = gp.sequences[seq_idx] + '.yuv'
sequence_dir = gp.generate_dir_path(gp.sequence_dir, gp.sequences[seq_idx])
if not os.path.exists(sequence_dir):
gp.create_dir(sequence_dir)
if not os.path.exists(sequence_dir + ori_sequence):
if gp.cur_platform == 'Linux':
os_system('cp ' + gp.sequence_dir + gp.sequences[seq_idx] +
'.yuv ' + sequence_dir)
else:
os_system('unzip ' + gp.sequence_dir + gp.sequences[seq_idx] +
'.yuv.zip -d ' + sequence_dir)
tmp = ori_sequence.split('_')
ori_width = int(tmp[-2].split('x')[0])
ori_height = int(tmp[-2].split('x')[1].split('p')[0])
ori_fps = int(tmp[-2].split('x')[1].split('p')[1])
for seq_name in gp.seq_candidates[seq_idx]:
tmp_seq_name = seq_name.split('_')
width = int(tmp_seq_name[3].split('p')[0].split('x')[0])
height = int(tmp_seq_name[3].split('p')[0].split('x')[1])
fps = int(tmp_seq_name[3].split('p')[1])
scale_seq_name = ori_sequence
scale_width = ori_width
scale_height = ori_height
if width * ori_height != height * ori_width:
if width * 1.0 / height < ori_width * 1.0 / ori_height:
scale_width = ori_height * width / height
scale_height = ori_height
else:
scale_width = ori_width
scale_height = ori_width * height / width
scale_seq_name = tmp[0] + '_' + tmp[1] + '_' + tmp[2] + '_' \
+ str(scale_width) + 'x' + str(scale_height) + 'p' \
+ str(ori_fps) + '_' + tmp[4]
if not os.path.exists(sequence_dir + scale_seq_name):
os_system(gp.scale.get_full_path_executable() + ' ' +
str(ori_width) + ' ' + str(ori_height) + ' ' +
str(scale_width) + ' ' + str(scale_height) + ' ' +
sequence_dir + ori_sequence + ' ' + sequence_dir +
scale_seq_name + ' 4 1')
gp.print_log(gp.LogLevel.Normal, 'Scaling ' + scale_seq_name)
if not os.path.exists(sequence_dir + seq_name):
os_system(gp.scale.get_full_path_executable() + ' ' +
str(scale_width) + ' ' + str(scale_height) + ' ' +
str(width) + ' ' + str(height) + ' ' + sequence_dir +
scale_seq_name + ' ' + sequence_dir + seq_name + ' 3 ' +
str(ori_fps) + ' ' + str(fps))
gp.print_log(gp.LogLevel.Normal, 'Scaling ' + seq_name)
def gen_dir(scenario):
gp.print_log(gp.LogLevel.Normal, 'Generating directories...')
gp.print_log(gp.LogLevel.Normal, 'Creating necessary folders...')
result_secnario_dir = gp.generate_dir_path(gp.result_dir, scenario)
gp.create_dir(result_secnario_dir)
if gp.connection_type == gp.connection[1]:
ref_log_scenario_dir = gp.generate_dir_path(gp.ref_log_dir, scenario)
gp.create_dir(ref_log_scenario_dir)
if gp.mode == 'Overnight' and not os.path.isdir(gp.backup_log_dir):
gp.create_dir(gp.backup_log_dir)
if gp.mode == 'Regression':
if gp.connection_type == gp.connection[0]:
gp.client.set_executable_dir(
gp.generate_dir_path(gp.executable_dir, 'anchor'))
else:
gp.client.set_executable_dir(
gp.generate_dir_path(gp.executable_dir, 'test'))
else:
gp.client.set_executable_dir(
gp.generate_dir_path(gp.executable_dir, 'overnight'))
def analyse_encoder_for_one_room(room_dir, room):
output = ''
client_list = os.listdir(room_dir)
for client in client_list:
output += room + gp.string_join + client + '\n'
client_dir = gp.generate_dir_path(room_dir, client)
if os.path.isdir(client_dir):
anchor_list = os.listdir(client_dir)
for anchor_file in anchor_list:
if re.search('enc_offline_test_0', anchor_file):
output += analyse_encoder_for_one_client(client_dir +
anchor_file)
break
else:
gp.print_log(gp.LogLevel.Normal,
'Error: Folders do not exist! ' + room_dir + client)
exit()
return output
def analyse_data(scenario, anchor, test):
anchor_dir = gp.generate_dir_path(anchor, scenario)
test_dir = gp.generate_dir_path(test, scenario)
is_match = True
if os.path.isdir(test_dir) and os.path.isdir(anchor_dir):
room_list = os.listdir(test_dir)
for room in room_list:
test_room_dir = gp.generate_dir_path(test_dir, room)
anchor_room_dir = gp.generate_dir_path(anchor_dir, room)
if os.path.isdir(test_room_dir) and os.path.isdir(anchor_room_dir):
uid_list = os.listdir(test_room_dir)
is_room_totally_match = True
for uid in uid_list:
test_uid_dir = gp.generate_dir_path(test_room_dir, uid)
anchor_uid_dir = gp.generate_dir_path(anchor_room_dir, uid)
if not os.path.isdir(test_uid_dir):
continue
case = uid.split(gp.string_join)
case[0] = int(case[0])
if os.path.isdir(test_uid_dir) and os.path.isdir(
anchor_uid_dir):
result = analyse_encoder_data(anchor_uid_dir,
test_uid_dir)
is_match = is_match & result
is_room_totally_match = is_room_totally_match & result
doc_list = os.listdir(test_room_dir)
for doc in doc_list:
if re.search('crash.txt', doc):
is_room_totally_match = False
is_match = False
doc_list = os.listdir(anchor_room_dir)
for doc in doc_list:
if re.search('crash.txt', doc):
is_room_totally_match = False
is_match = False
if is_room_totally_match:
gp.remove_dir(test_room_dir)
gp.remove_dir(anchor_room_dir)
if is_match == 0:
gp.print_log(gp.LogLevel.Normal, '\nMismatch Detected!!!\n')
else:
gp.print_log(gp.LogLevel.Normal, '\nAll Match!!!\n')
return is_match
def analyse_decoder_for_one_room(room_dir, room):
output = ''
client_list = os.listdir(room_dir)
for client in client_list:
output += room + gp.string_join + client + '\n'
client_dir = gp.generate_dir_path(room_dir, client)
if os.path.isdir(client_dir):
anchor_list = os.listdir(client_dir)
result = []
for anchor_file in anchor_list:
if re.search('dec_offline_test', anchor_file):
result.append(
analyse_decoder_for_one_client(client_dir +
anchor_file))
for data in result:
found = False
for anchor_file in anchor_list:
if re.search('vqmg_' + data[1][1] + '.', anchor_file):
vqmg_file = client_dir + anchor_file
fp = open(vqmg_file, 'r')
fp.readline()
tmp = fp.readline().split('\n')[0].split('\t')[-1]
fp.close()
data[1][0] = data[1][0] + 'vqmg\t' + tmp + '\n'
found = True
if not found:
data[1][0] = data[1][0] + 'vqmg\t0\n'
result.sort()
for idx in range(0, len(result)):
output += 'For' + gp.string_join + 'UID' + gp.string_join + str(
result[idx][0]) + '\n'
output += result[idx][1][0]
else:
gp.print_log(gp.LogLevel.Normal,
'Error: Folders do not exist! ' + client_dir)
exit()
return output
def find_latest_ref_log(commit_id):
gp.ref_log_dir = ''
ref_file = ''
if os.path.isdir(gp.backup_log_dir):
zip_list = os.listdir(gp.backup_log_dir)
time_commit = 0
time_run = 0
for zip_file in zip_list:
tmp = zip_file.split('.')[0].split(gp.folder_join)
# to avoid .DS_Store file problem
if tmp[0] != '':
time_run_tmp = int(tmp[0])
time_commit_id_tmp = tmp[1]
time_commit_tmp = int(tmp[2])
if commit_id != '':
if re.search(
commit_id,
time_commit_id_tmp) and time_run < time_run_tmp:
time_run = time_run_tmp
ref_file = zip_file
else:
if time_commit < time_commit_tmp:
if time_run < time_run_tmp:
time_run = time_run_tmp
time_commit = time_commit_tmp
commit_id = tmp[1]
ref_file = zip_file
if ref_file != '':
gp.print_log(gp.LogLevel.Normal, 'Selected commit id is ' + commit_id)
os.system('unzip -oq ' + gp.backup_log_dir + ref_file + ' -d ' +
gp.data_dir)
gp.ref_log_dir = gp.generate_dir_path(gp.data_dir,
ref_file.split('.')[0])
return commit_id[0:7]
gp.print_log(gp.LogLevel.Normal, 'No commit is available!!!')
return ''
def generate_data(scenario):
enc_result_file = open(
gp.cur_log_dir + 'Enc_File_' + scenario + '_' + gp.cur_time + '_' +
gp.cur_commit_id + '.txt', 'w')
dec_result_file = open(
gp.cur_log_dir + 'Dec_File_' + scenario + '_' + gp.cur_time + '_' +
gp.cur_commit_id + '.txt', 'w')
anchor_dir = gp.cur_log_dir + scenario
if os.path.isdir(anchor_dir):
room_list = os.listdir(anchor_dir)
for room in room_list:
room_dir = gp.generate_dir_path(anchor_dir, room)
if os.path.isdir(room_dir):
# Encoder result analyse
enc_result = analyse_encoder_for_one_room(room_dir, room)
enc_result_file.write(enc_result)
# Decoder result analyse
dec_result = analyse_decoder_for_one_room(room_dir, room)
dec_result_file.write(dec_result)
enc_result_file.close()
dec_result_file.close()
def saving_logs(scenario):
result_dir = gp.generate_dir_path(gp.result_dir, scenario)
if gp.connection_type == gp.connection[0]:
dst_dir = gp.generate_dir_path(gp.cur_log_dir, scenario)
os.system('cp ' + gp.client.dir_ + 'commit.log ' + gp.cur_log_dir)
else:
dst_dir = gp.generate_dir_path(gp.ref_log_dir, scenario)
os.system('cp ' + gp.client.dir_ + 'commit.log ' + gp.ref_log_dir)
room_list = os.listdir(result_dir)
for room in room_list:
if os.path.isdir(result_dir + room):
uid_list = os.listdir(result_dir + room)
room_dir = gp.generate_dir_path(result_dir, room)
for uid in uid_list:
if os.path.isdir(room_dir + uid):
cur_uid_dir = gp.generate_dir_path(room_dir, uid)
doc_list = os.listdir(cur_uid_dir)
dst_uid_dir = gp.generate_dir_path(dst_dir, room, uid)
gp.create_dir(dst_uid_dir)
for doc in doc_list:
if re.search('enc_offline_test_0', doc) \
or re.search('enc_online_parameters', doc) \
or re.search('enc_save_sent_stream', doc) \
or re.search('enc_save_stream_info_0', doc):
os.system('cp ' + cur_uid_dir + doc + ' ' +
dst_uid_dir)
if gp.mode == 'Overnight':
if re.search('dec_offline_test', doc) \
or re.search('dec_save_stream_info', doc) \
or re.search('dec_quality_score', doc) \
or re.search('dec_save_stream_received', doc) \
or re.search('vqmg', doc) \
or re.search('crash', doc) \
or re.search('timestamp', doc):
os.system('cp ' + cur_uid_dir + doc + ' ' +
dst_uid_dir)
def compare_data():
if gp.cur_log_dir[-1] != '/':
gp.cur_log_dir += '/'
if gp.ref_log_dir[-1] != '/':
gp.ref_log_dir += '/'
ref_list = os.listdir(gp.ref_log_dir)
cur_list = os.listdir(gp.cur_log_dir)
ref_folder = gp.ref_log_dir.split('/')[-2]
cur_folder = gp.cur_log_dir.split('/')[-2]
problem_zip_name = gp.cur_time + gp.folder_join + cur_folder.split(gp.folder_join)[1][0:7] \
+ '_vs_' + ref_folder.split(gp.folder_join)[1][0:7]
gp.problem_case_dir = gp.generate_dir_path(gp.data_dir, problem_zip_name)
gp.create_dir(gp.problem_case_dir)
if not os.path.isdir(gp.problem_dir):
gp.create_dir(gp.problem_dir)
header = '<h1>\nThis email is the overnight comparison results\n</h1>'
header += '<h2>\n Current Commit:\n</h2>' + gp.read_commit_log(gp.cur_log_dir) + \
'<h3>Run on ' + gp.convert_date(cur_folder.split(gp.folder_join)[0]) + '</h3>'
header += '<h2>\n Ref Commit:\n</h2>' + gp.read_commit_log(gp.ref_log_dir) + \
'<h3>Run on ' + gp.convert_date(ref_folder.split(gp.folder_join)[0]) + '</h3>'
if gp.total_crash != 0:
header += '<h2>\n<span style="color: red"> Total Crash: ' + str(
gp.total_crash) + '</span>\n</h2>'
else:
header += '<h2>\n No Crash.\n</h2>'
result = ''
#brief-result
brief_result = ''
for scenario in gp.scenario:
result += '<hr>\n'
found_case = True
ref_enc_file = find_file('Enc_File_' + scenario, ref_list)
ref_dec_file = find_file('Dec_File_' + scenario, ref_list)
if ref_enc_file != '' and ref_dec_file != '':
ref_case_set = cc.CaseSummary(gp.ref_log_dir + ref_enc_file,
gp.ref_log_dir + ref_dec_file)
else:
found_case = False
cur_enc_file = find_file('Enc_File_' + scenario, cur_list)
cur_dec_file = find_file('Dec_File_' + scenario, cur_list)
if cur_enc_file != '' and cur_dec_file != '':
cur_case_set = cc.CaseSummary(gp.cur_log_dir + cur_enc_file,
gp.cur_log_dir + cur_dec_file)
else:
found_case = False
if found_case:
result += compare_encoder_performance(cur_case_set, ref_case_set,
scenario)
result += compare_decoder_performance(cur_case_set, ref_case_set,
scenario)
brief_result += compare_encoder_performance_brief(scenario)
brief_result += compare_decoder_performance_brief(scenario)
if gp.total_mismatch != 0:
header += '<h2>\n<span style="color: red"> Total Mismatch: ' + str(
gp.total_mismatch) + '</span>\n</h2>'
else:
header += '<h2>\n No Mismatch.\n</h2>'
output = '<html>\n<head>\n' + header + result + '</head>\n</html>\n'
brief_output = '<html>\n<head>\n' + header + brief_result + '</head>\n</html>\n'
result_file = cur_folder.split(
gp.folder_join)[1][:7] + '_vs_' + ref_folder.split(
gp.folder_join)[1][:7] + '.html'
result_file_handle = open(gp.problem_case_dir + result_file, 'w')
result_file_handle.write(output)
result_file_handle.close()
brief_result_file = cur_folder.split(
gp.folder_join)[1][:7] + '_vs_' + ref_folder.split(
gp.folder_join)[1][:7] + '_brief.html'
brief_result_file_handle = open(gp.problem_case_dir + brief_result_file,
'w')
brief_result_file_handle.write(brief_output)
brief_result_file_handle.close()
gp.move_to_dir(gp.pic_dir, gp.problem_case_dir)
gp.zip_to_folder(problem_zip_name + '.zip', gp.problem_case_dir,
gp.problem_dir)
gp.remove_dir(gp.problem_case_dir)
return output
def run_one_scenario(scenario):
"""
Only run either Comm/Live/ScSh mode
"""
fun.gen_dir(scenario)
fun.check_files()
p = multiprocessing.Pool()
gp.print_log(gp.LogLevel.Normal, 'Scaling YUVs...')
for seqIdx in gp.sequences:
temp = (seqIdx, )
p.apply_async(fun.scale_yuv, temp)
# fun.scale_yuv(seqIdx)
p.close()
p.join()
gp.print_log(gp.LogLevel.Normal, '')
gp.client_flag = gp.mgr.list(
[gp.RunningState.Unfinished for _ in range(len(gp.clients))])
p = multiprocessing.Pool()
gp.print_log(gp.LogLevel.Normal, 'Start offline test...')
while 1:
# Find a unfinished cases
gp.process_lock.acquire()
[all_start, all_finish, start_idx, end_idx] = fun.get_next_case()
gp.process_lock.release()
client_num = end_idx - start_idx
if all_finish:
break
if all_start:
time.sleep(1)
continue
require_capacity = 0
for client_idx in range(start_idx, end_idx):
client = gp.clients[client_idx]
assert isinstance(client, fun.Client)
require_capacity += client.capacity_
# Hold to wait enough processors
while 1:
gp.process_lock.acquire()
if gp.running_process.value + client_num <= gp.active_process \
and gp.used_capacity.value + require_capacity <= gp.capacity:
gp.running_process.value += client_num
gp.used_capacity.value += require_capacity
gp.process_lock.release()
break
elif gp.used_capacity.value == 0 and require_capacity > gp.capacity:
gp.print_log(
gp.LogLevel.Normal,
"Not enough capacity to run this case: " +
gp.clients[start_idx].case_)
gp.process_lock.release()
time.sleep(1)
if gp.connection_type == gp.connection[0] and os.popen(
'pgrep ' + gp.server.get_executable_name()).read() == '':
os.system(gp.server.get_full_path_executable() + ' >>/dev/null &')
gp.print_log(
gp.LogLevel.Normal, 'Restart Server! Current Server PID is ' +
os.popen('pgrep ' + gp.server.get_executable_name()).read())
for client_idx in range(start_idx, end_idx):
gp.process_lock.acquire()
gp.client_flag[client_idx] = gp.RunningState.Running
gp.process_lock.release()
client = gp.clients[client_idx]
assert isinstance(client, fun.Client)
uid_dir_name = str(client.uid_) + gp.string_join + client.config_
client_dir = gp.generate_dir_path(gp.result_dir, scenario,
client.case_, uid_dir_name)
gp.create_dir(client_dir)
fun.os_system('cp ' + gp.network_dir +
gp.networks[client.network_] + ' ' + client_dir)
gp.client.copy_executable_to_dir(client_dir)
if gp.connection_type == gp.connection[1]:
log_dir = gp.generate_dir_path(gp.cur_log_dir, scenario,
client.case_, uid_dir_name)
fun.os_system('cp ' + log_dir + 'enc_online_parameters* ' +
client_dir)
gp.print_log(
gp.LogLevel.Normal, 'Running Case ' + client.case_ +
' Client with UID ' + str(client.uid_) + ', Duration ' +
str(client.duration_) + ', Time ' + time.strftime(
'%Y_%m_%d_%H_%M_%S', time.localtime(time.time())))
p.apply_async(fun.run_client,
args=(client_idx, client_dir, scenario))
# fun.run_client(client_idx, client_dir, scenario)
gp.print_log(gp.LogLevel.Normal, '')
p.close()
p.join()
if gp.mode == 'Overnight':
gp.client_flag = gp.mgr.list(
[gp.RunningState.Unfinished for _ in range(len(gp.clients))])
p = multiprocessing.Pool()
gp.print_log(gp.LogLevel.Normal, 'Start VQM test...')
while 1:
# Find a unfinished cases
gp.process_lock.acquire()
[all_start, all_finish, start_idx, end_idx] = fun.get_next_case()
gp.process_lock.release()
if all_finish:
break
if all_start:
time.sleep(1)
continue
# Hold to wait enough processors
for client_idx in range(start_idx, end_idx):
while 1:
gp.process_lock.acquire()
if gp.running_process.value < gp.active_process / 2:
gp.running_process.value += 1
gp.process_lock.release()
break
gp.process_lock.release()
time.sleep(1)
gp.process_lock.acquire()
gp.client_flag[client_idx] = gp.RunningState.Running
gp.process_lock.release()
client = gp.clients[client_idx]
assert isinstance(client, fun.Client)
uid_dir_name = str(
client.uid_) + gp.string_join + client.config_
client_dir = gp.generate_dir_path(gp.result_dir, scenario,
client.case_, uid_dir_name)
gp.vqm_test.copy_executable_to_dir(client_dir)
gp.decode_stream.copy_executable_to_dir(client_dir)
gp.scale.copy_executable_to_dir(client_dir)
gp.print_log(
gp.LogLevel.Normal, 'Running VQM Case ' + client.case_ +
' Client with UID ' + str(client.uid_) + ', Duration ' +
str(client.duration_) + ', Time ' + time.strftime(
'%Y_%m_%d_%H_%M_%S', time.localtime(time.time())))
p.apply_async(fun.run_vqm, args=(client_idx, client_dir))
# fun.run_vqm(client_idx, client_dir)
gp.print_log(gp.LogLevel.Normal, '')
p.close()
p.join()
fun.saving_logs(scenario)
if gp.scenario_crash.value != 0:
gp.print_log(
gp.LogLevel.Normal, 'Total ' + str(gp.scenario_crash.value) +
' crashes in for ' + scenario + '!!!')
gp.total_crash += gp.scenario_crash.value
gp.scenario_crash.value = 0
