def test_with_sample_input(self):
""" User passes correct args, should pass
"""
raised = False
config_path = os.path.join(tool_base, 'config.ini')
config = configparser.ConfigParser()
config.read(config_path)
username = config[WCS_CREDS_SECTION][WCS_USERNAME_ITEM]
password = config[WCS_CREDS_SECTION][WCS_PASSWORD_ITEM]
intent_path = os.path.join(tool_base, 'resources', 'sample',
'intents.csv')
entity_path = os.path.join(tool_base, 'resources', 'sample',
'entities.csv')
train_args = [
sys.executable, TRAIN_CONVERSATION_PATH, '-i', intent_path, '-e',
entity_path, '-u', username, '-p', password
]
workspace_spec_json = os.path.join(self.test_dir, SPEC_FILENAME)
print('Begin training')
returncode = 0
with open(workspace_spec_json, 'w+') as f:
returncode = subprocess.run(train_args, stdout=f).returncode
try:
with open(workspace_spec_json, 'r') as f:
if returncode != 0:
print('Training failed')
print(f.read())
raise Exception()
else:
print('Training complete')
workspace_id = json.load(f)[WORKSPACE_ID_TAG]
test_in_path = os.path.join(tool_base, 'resources',
'sample', 'test.csv')
test_out_path = os.path.join(self.test_dir, 'test-out.csv')
test_args = [
sys.executable, self.script_path, '-i', test_in_path,
'-o', test_out_path, '-w', workspace_id, '-t',
'utterance', '-m', '-g', 'golden intent', '-u',
username, '-p', password, '-r',
str(DEFAULT_TEST_RATE)
]
print('Begin testing')
returncode = subprocess.run(test_args).returncode
delete_workspaces(username, password, [workspace_id])
if returncode != 0:
print('Testing failed')
raise Exception()
print('Testing completed')
except Exception as e:
print(e)
raised = True
self.assertFalse(raised, 'Exception raised')
def test_std(self):
""" User passes correct standard test config, should pass
"""
raised = False
std_test_dir = os.path.join(self.test_dir, STANDARD_TEST)
if not os.path.exists(std_test_dir):
os.makedirs(std_test_dir)
# Use fold 0 training input as intent
intent_path = os.path.join(tool_base, 'resources', 'sample', 'kfold',
'0', 'train.csv')
entity_path = os.path.join(tool_base, 'resources', 'sample',
'entities.csv')
username = self.config[WCS_CREDS_SECTION][WCS_USERNAME_ITEM]
password = self.config[WCS_CREDS_SECTION][WCS_PASSWORD_ITEM]
args = [
sys.executable, TRAIN_CONVERSATION_PATH, '-i', intent_path, '-e',
entity_path, '-u', username, '-p', password
]
workspace_spec_json = os.path.join(self.test_dir, SPEC_FILENAME)
# Train a new instance in order to pull the workspace detail
print('Begin training for setting up environment')
with open(workspace_spec_json, 'w+') as f:
if subprocess.run(args, stdout=f).returncode != 0:
print('Training failed')
print(f.read())
raise Exception()
with open(workspace_spec_json, 'r') as f:
self.config[DEFAULT_SECTION][WORKSPACE_ID_ITEM] = \
json.load(f)[WORKSPACE_ID_TAG]
test_input_path = os.path.join(tool_base, 'resources', 'sample',
'kfold', '0', 'test.csv')
self.config[DEFAULT_SECTION][MODE_ITEM] = STANDARD_TEST
self.config[DEFAULT_SECTION][TEMP_DIR_ITEM] = std_test_dir
self.config[DEFAULT_SECTION][TEST_FILE_ITEM] = test_input_path
self.config[DEFAULT_SECTION][TEST_OUT_PATH_ITEM] = \
os.path.join(std_test_dir, 'test-out.csv')
std_config_path = os.path.join(std_test_dir, 'config.ini')
with open(std_config_path, 'w') as configfile:
self.config.write(configfile)
args = [sys.executable, self.script_path, '-c', std_config_path]
if subprocess.run(args).returncode != 0:
raised = True
delete_workspaces(username, password,
[self.config[DEFAULT_SECTION][WORKSPACE_ID_ITEM]])
self.assertFalse(raised, 'Exception raised')
def test_kfold(self):
""" User passes correct kfold config, should pass
"""
raised = False
kfold_test_dir = os.path.join(self.test_dir, KFOLD)
if not os.path.exists(kfold_test_dir):
os.makedirs(kfold_test_dir)
intent_path = os.path.join(tool_base, 'resources', 'sample',
'intents.csv')
entity_path = os.path.join(tool_base, 'resources', 'sample',
'entities.csv')
username = self.config[WCS_CREDS_SECTION][WCS_USERNAME_ITEM]
password = self.config[WCS_CREDS_SECTION][WCS_PASSWORD_ITEM]
args = [
sys.executable, TRAIN_CONVERSATION_PATH, '-i', intent_path, '-e',
entity_path, '-u', username, '-p', password
]
workspace_spec_json = os.path.join(self.test_dir, SPEC_FILENAME)
# Train a new instance in order to pull the workspace detail
print('Begin training for setting up environment')
with open(workspace_spec_json, 'w+') as f:
if subprocess.run(args, stdout=f).returncode != 0:
print('Training failed')
print(f.read())
raise Exception()
with open(workspace_spec_json, 'r') as f:
self.config[DEFAULT_SECTION][WORKSPACE_ID_ITEM] = \
json.load(f)[WORKSPACE_ID_TAG]
self.config[DEFAULT_SECTION][MODE_ITEM] = KFOLD
self.config[DEFAULT_SECTION][TEMP_DIR_ITEM] = kfold_test_dir
self.config[DEFAULT_SECTION][FIGURE_PATH_ITEM] = \
os.path.join(kfold_test_dir, 'figure.png')
self.config[DEFAULT_SECTION][FOLD_NUM_ITEM] = str(FOLD_NUM_DEFAULT)
self.config[DEFAULT_SECTION][PARTIAL_CREDIT_TABLE_ITEM] = \
os.path.join(tool_base, 'resources', 'sample',
'partial-credit-table.csv')
kfold_config_path = os.path.join(kfold_test_dir, 'config.ini')
with open(kfold_config_path, 'w') as configfile:
self.config.write(configfile)
args = [sys.executable, self.script_path, '-c', kfold_config_path]
if subprocess.run(args).returncode != 0:
raised = True
delete_workspaces(username, password,
[self.config[DEFAULT_SECTION][WORKSPACE_ID_ITEM]])
self.assertFalse(raised, 'Exception raised')
def test(out_dir, intent_train_file, workspace_base_file, test_out_path,
test_input_file, keep_workspace, username, password, iam_apikey, version, url):
print('Begin {} with following details:'.format(STANDARD_TEST.upper()))
print('{}={}'.format(INTENT_FILE_ITEM, intent_train_file))
print('{}={}'.format(WORKSPACE_BASE_ITEM, workspace_base_file))
print('{}={}'.format(TEST_FILE_ITEM, test_input_file))
print('{}={}'.format(TEST_OUT_PATH_ITEM, test_out_path))
print('{}={}'.format(OUT_DIR_ITEM, out_dir))
print('{}={}'.format(DO_KEEP_WORKSPACE_ITEM, BOOL_MAP[keep_workspace]))
print('{}={}'.format(WCS_USERNAME_ITEM, username))
print('{}={}'.format(WCS_BASEURL_ITEM, url))
print('{}={}'.format(WA_API_VERSION_ITEM, version))
# Validate test file
extra_params = []
test_input = pd.read_csv(test_input_file, quoting=csv.QUOTE_ALL,
encoding=UTF_8)
header = list(test_input.columns.values)
if UTTERANCE_COLUMN in header:
extra_params += ['-t', UTTERANCE_COLUMN]
else:
if len(header) != 1:
raise ValueError('Test input has unknown utterance column')
# Run standard test
working_dir = os.path.join(out_dir, STANDARD_TEST)
if not os.path.exists(working_dir):
os.makedirs(working_dir)
workspace_spec_json = os.path.join(working_dir, SPEC_FILENAME)
train_args = [sys.executable, TRAIN_CONVERSATION_PATH,
'-i', intent_train_file,
'-n', 'standard test', '-v', version,
'-u', username, '-p', password, '-a', iam_apikey, '-l', url,
'-w', workspace_base_file]
with open(workspace_spec_json, 'w') as f:
if subprocess.run(train_args, stdout=f).returncode == 0:
print('Trained standard test workspace')
else:
raise RuntimeError('Failure in training workspace')
workspace_id = None
with open(workspace_spec_json, 'r') as f:
workspace_id = json.load(f)[WORKSPACE_ID_TAG]
try:
if subprocess.run([sys.executable, TEST_CONVERSATION_PATH,
'-i', test_input_file,
'-o', test_out_path, '-m', '-v', version,
'-u', username, '-p', password, '-a', iam_apikey, '-l', url,
'-w', workspace_id,
'-r', str(MAX_TEST_RATE)] + extra_params
).returncode == 0:
print('Tested workspace')
else:
raise RuntimeError('Failure in testing data')
finally:
if not keep_workspace:
delete_workspaces(username, password, iam_apikey, url, version, [workspace_id])
def test_with_sample_input(self):
""" User passes correct args, should pass
"""
raised = False
intent_path = os.path.join(tool_base, 'resources', 'sample',
'intents.csv')
entity_path = os.path.join(tool_base, 'resources', 'sample',
'entities.csv')
config_path = os.path.join(tool_base, 'config.ini')
config = configparser.ConfigParser()
config.read(config_path)
username = config[WCS_CREDS_SECTION][WCS_USERNAME_ITEM]
password = config[WCS_CREDS_SECTION][WCS_PASSWORD_ITEM]
args = [
sys.executable, TRAIN_CONVERSATION_PATH, '-i', intent_path, '-e',
entity_path, '-u', username, '-p', password
]
workspace_spec_json = os.path.join(self.test_dir, SPEC_FILENAME)
# Train a new instance in order to pull the workspace detail
print('Begin training for setting up environment')
with open(workspace_spec_json, 'w+') as f:
if subprocess.run(args, stdout=f).returncode != 0:
print('Training failed')
print(f.read())
raise Exception()
workspace_id = None
with open(workspace_spec_json, 'r') as f:
workspace_id = \
json.load(f)[WORKSPACE_ID_TAG]
args = \
self.parser.parse_args(
['-w', workspace_id, '-u', username,
'-p', password, '-o', self.test_dir])
try:
workspaceParser.func(args)
except Exception as e:
print(e)
raised = True
delete_workspaces(username, password, [workspace_id])
self.assertFalse(raised, 'Exception raised')
def kfold(fold_num, out_dir, intent_train_file, workspace_base_file,
figure_path, keep_workspace, username, password, iam_apikey, url, version, weight_mode,
conf_thres, partial_credit_table):
FOLD_TRAIN = 'fold_train'
FOLD_TEST = 'fold_test'
WORKSPACE_SPEC = 'fold_workspace'
WORKSPACE_NAME = 'workspace_name'
TEST_OUT = 'test_out'
print('Begin {} with following details:'.format(KFOLD.upper()))
print('{}={}'.format(INTENT_FILE_ITEM, intent_train_file))
print('{}={}'.format(WORKSPACE_BASE_ITEM, workspace_base_file))
print('{}={}'.format(FIGURE_PATH_ITEM, figure_path))
print('{}={}'.format(OUT_DIR_ITEM, out_dir))
print('{}={}'.format(FOLD_NUM_ITEM, fold_num))
print('{}={}'.format(DO_KEEP_WORKSPACE_ITEM, BOOL_MAP[keep_workspace]))
print('{}={}'.format(WEIGHT_MODE_ITEM, weight_mode))
print('{}={}'.format(CONF_THRES_ITEM, conf_thres))
print('{}={}'.format(WCS_USERNAME_ITEM, username))
print('{}={}'.format(WCS_BASEURL_ITEM, url))
print('{}={}'.format(WA_API_VERSION_ITEM, version))
print('{}={}'.format(PARTIAL_CREDIT_TABLE_ITEM, partial_credit_table))
working_dir = os.path.join(out_dir, KFOLD)
if not os.path.exists(working_dir):
os.makedirs(working_dir)
# Prepare folds
if subprocess.run([sys.executable, CREATE_TEST_TRAIN_FOLDS_PATH,
'-i', intent_train_file, '-o', working_dir,
'-k', str(fold_num)],
stdout=subprocess.PIPE).returncode == 0:
print('Created {} folds'.format(str(fold_num)))
else:
raise RuntimeError('Failure in folds creation')
# Construct fold params
fold_params = [{FOLD_TRAIN: os.path.join(working_dir, str(idx),
TRAIN_FILENAME),
FOLD_TEST: os.path.join(working_dir, str(idx),
TEST_FILENAME),
TEST_OUT: os.path.join(working_dir, str(idx),
TEST_OUT_FILENAME),
WORKSPACE_SPEC: os.path.join(working_dir,
str(idx), SPEC_FILENAME),
WORKSPACE_NAME: '{}_{}'.format(KFOLD, str(idx))}
for idx in range(fold_num)]
# Begin training
train_processes_specs = {}
for fold_param in fold_params:
spec_file = open(fold_param[WORKSPACE_SPEC], 'w')
train_args = [sys.executable, TRAIN_CONVERSATION_PATH,
'-i', fold_param[FOLD_TRAIN],
'-n', fold_param[WORKSPACE_NAME],
'-u', username, '-p', password,
'-a', iam_apikey,
'-l', url, '-v', version,
'-w', workspace_base_file]
train_processes_specs[
subprocess.Popen(train_args, stdout=spec_file)] = spec_file
train_failure_idx = []
for idx, (process, file) in enumerate(train_processes_specs.items()):
if process.wait() == 0:
file.close()
else:
train_failure_idx.append(idx)
try:
if len(train_failure_idx) != 0:
raise RuntimeError(
'Fail to train {} fold workspace'.format(','.join(
str(train_failure_idx))))
print('Trained {} workspaces'.format(str(fold_num)))
# Begin testing
test_processes = []
workspace_ids = []
FOLD_TEST_RATE = int(MAX_TEST_RATE / fold_num)
for fold_param in fold_params:
workspace_id = None
with open(fold_param[WORKSPACE_SPEC]) as f:
workspace_id = json.load(f)[WORKSPACE_ID_TAG]
workspace_ids.append(workspace_id)
test_args = [sys.executable, TEST_CONVERSATION_PATH,
'-i', fold_param[FOLD_TEST],
'-o', fold_param[TEST_OUT],
'-u', username, '-p', password,
'-a', iam_apikey, '-l', url, '-v', version,
'-t', UTTERANCE_COLUMN, '-g', GOLDEN_INTENT_COLUMN,
'-w', workspace_id, '-r', str(FOLD_TEST_RATE),
'-m']
if partial_credit_table is not None:
test_args += ['--partial_credit_table', partial_credit_table]
test_processes.append(subprocess.Popen(test_args))
test_failure_idx_str = []
for idx, process in enumerate(test_processes):
if process.wait() != 0:
test_failure_idx_str.append(str(idx))
if len(test_failure_idx_str) != 0:
raise RuntimeError('Fail to test {} fold workspace'.format(
','.join(test_failure_idx_str)))
print('Tested {} workspaces'.format(str(fold_num)))
test_out_files = [fold_param[TEST_OUT] for fold_param in fold_params]
# Add a column for the fold number
for idx, this_file in enumerate(test_out_files):
this_df = pd.read_csv(this_file, quoting=csv.QUOTE_ALL, encoding='utf-8', \
keep_default_na=False)
this_df['Fold Index'] = idx
this_df.to_csv( this_file, encoding='utf-8', quoting=csv.QUOTE_ALL, index=False )
# Union test out
kfold_result_file = os.path.join(out_dir, KFOLD_UNION_FILE)
pd.concat([pd.read_csv(file, quoting=csv.QUOTE_ALL, encoding=UTF_8,
keep_default_na=False)
for file in test_out_files]) \
.to_csv(kfold_result_file,
encoding='utf-8', quoting=csv.QUOTE_ALL, index=False)
print("Wrote k-fold result file to {}".format(kfold_result_file))
classfier_names = ['Fold {}'.format(idx) for idx in range(fold_num)]
plot_args = [sys.executable, CREATE_PRECISION_CURVE_PATH,
'-t', '{} Fold Test'.format(str(fold_num)),
'-o', figure_path, '-w', weight_mode,
'--tau', conf_thres, '-n'] + \
classfier_names + ['-i'] + test_out_files
if subprocess.run(plot_args).returncode != 0:
raise RuntimeError('Failure in plotting curves')
kfold_result_file_base = kfold_result_file[:-4]
metrics_args = [sys.executable, INTENT_METRICS_PATH,
'-i', kfold_result_file,
'-o', kfold_result_file_base+".metrics.csv",
'--partial_credit_on', str(partial_credit_table is not None)]
if subprocess.run(metrics_args).returncode != 0:
raise RuntimeError('Failure in generating intent metrics')
confusion_args = [sys.executable, CONFUSION_MATRIX_PATH,
'-i', kfold_result_file,
'-o', kfold_result_file_base+".confusion_args.csv"]
if subprocess.run(confusion_args).returncode != 0:
raise RuntimeError('Failure in generating confusion matrix')
finally:
if not keep_workspace:
workspace_ids = []
for idx in range(fold_num):
if idx not in train_failure_idx:
with open(fold_params[idx][WORKSPACE_SPEC]) as f:
workspace_id = json.load(f)[WORKSPACE_ID_TAG]
workspace_ids.append(workspace_id)
delete_workspaces(username, password, iam_apikey, url, version, workspace_ids)
def blind(out_dir, intent_train_file, workspace_base_file, figure_path,
test_out_path, test_input_file, previous_blind_out, keep_workspace,
username, password, iam_apikey, url, version, weight_mode, conf_thres, partial_credit_table, figure_title):
print('Begin {} with following details:'.format(BLIND_TEST.upper()))
print('{}={}'.format(INTENT_FILE_ITEM, intent_train_file))
print('{}={}'.format(WORKSPACE_BASE_ITEM, workspace_base_file))
print('{}={}'.format(TEST_FILE_ITEM, test_input_file))
print('{}={}'.format(PREVIOUS_BLIND_OUT_ITEM, previous_blind_out))
print('{}={}'.format(FIGURE_PATH_ITEM, figure_path))
print('{}={}'.format(TEST_OUT_PATH_ITEM, test_out_path))
print('{}={}'.format(OUT_DIR_ITEM, out_dir))
print('{}={}'.format(DO_KEEP_WORKSPACE_ITEM, BOOL_MAP[keep_workspace]))
print('{}={}'.format(WEIGHT_MODE_ITEM, weight_mode))
print('{}={}'.format(CONF_THRES_ITEM, conf_thres))
print('{}={}'.format(WCS_USERNAME_ITEM, username))
print('{}={}'.format(WCS_BASEURL_ITEM, url))
print('{}={}'.format(WA_API_VERSION_ITEM, version))
print('{}={}'.format(PARTIAL_CREDIT_TABLE_ITEM, partial_credit_table))
# Validate previous blind out format
test_out_files = [test_out_path]
classfier_names = ['New Classifier']
if previous_blind_out is not None:
df = pd.read_csv(previous_blind_out, quoting=csv.QUOTE_ALL,
encoding=UTF_8)
header = list(df.columns.values)
if INTENT_JUDGE_COLUMN not in header:
raise ValueError(
"'{}' column doesn't exist in {}.".format(
INTENT_JUDGE_COLUMN, previous_blind_out))
test_out_files.append(previous_blind_out)
classfier_names.append('Old Classifier')
working_dir = os.path.join(out_dir, BLIND_TEST)
if not os.path.exists(working_dir):
os.makedirs(working_dir)
workspace_spec_json = os.path.join(working_dir, SPEC_FILENAME)
train_args = [sys.executable, TRAIN_CONVERSATION_PATH,
'-i', intent_train_file, '-n', 'blind test',
'-u', username, '-p', password,
'-a', iam_apikey,
'-l', url, '-v', version,
'-w', workspace_base_file]
with open(workspace_spec_json, 'w') as f:
if subprocess.run(train_args, stdout=f).returncode == 0:
print('Trained blind workspace')
else:
raise RuntimeError('Failure in training workspace')
workspace_id = None
with open(workspace_spec_json, 'r') as f:
workspace_id = json.load(f)[WORKSPACE_ID_TAG]
try:
test_args = [sys.executable, TEST_CONVERSATION_PATH,
'-i', test_input_file,
'-o', test_out_path, '-m',
'-u', username, '-p', password, '-a', iam_apikey, '-l', url,
'-t', UTTERANCE_COLUMN, '-g', GOLDEN_INTENT_COLUMN,
'-w', workspace_id, '-v', version,
'-r', str(MAX_TEST_RATE)]
if partial_credit_table is not None:
test_args += ['--partial_credit_table', partial_credit_table]
if subprocess.run(test_args).returncode == 0:
print('Tested blind workspace')
else:
raise RuntimeError('Failure in testing blind data')
if subprocess.run([sys.executable, CREATE_PRECISION_CURVE_PATH,
'-t', figure_title, '-w', weight_mode, '--tau',
conf_thres, '-o', figure_path,
'-n'] + classfier_names +
['-i'] + test_out_files).returncode != 0:
raise RuntimeError('Failure in plotting curves')
blind_result_file = test_out_path
blind_result_file_base = blind_result_file[:-4]
metrics_args = [sys.executable, INTENT_METRICS_PATH,
'-i', blind_result_file,
'-o', blind_result_file_base+"_metrics.csv",
'--partial_credit_on', str(partial_credit_table is not None)]
if subprocess.run(metrics_args).returncode != 0:
raise RuntimeError('Failure in generating intent metrics')
confusion_args = [sys.executable, CONFUSION_MATRIX_PATH,
'-i', blind_result_file,
'-o', blind_result_file_base+"_confusion.csv"]
if subprocess.run(confusion_args).returncode != 0:
raise RuntimeError('Failure in generating confusion matrix')
finally:
if not keep_workspace:
delete_workspaces(username, password, iam_apikey, url, version, [workspace_id])
def test_kfold(self):
""" User passes correct kfold config, should pass
"""
print('=============================================')
print('test_kfold - Executes KFOLD tests with run.py')
print('=============================================')
raised = False
kfold_test_dir = os.path.join(self.test_dir, KFOLD)
if not os.path.exists(kfold_test_dir):
os.makedirs(kfold_test_dir)
intent_path = os.path.join(tool_base, 'resources', 'sample',
'intents.csv')
entity_path = os.path.join(tool_base, 'resources', 'sample',
'entities.csv')
apikey = self.config[WCS_CREDS_SECTION][WCS_IAM_APIKEY_ITEM]
args = [
sys.executable, TRAIN_CONVERSATION_PATH, '-i', intent_path, '-e',
entity_path, '-a', apikey, '-n', 'KFOLD_TEST_RUN', '-l',
self.config[WCS_CREDS_SECTION][WCS_BASEURL_ITEM]
]
workspace_spec_json = os.path.join(self.test_dir, SPEC_FILENAME)
# Train a new instance in order to pull the workspace detail
print('Begin training for setting up environment')
with open(workspace_spec_json, 'w+') as f:
if subprocess.run(args, stdout=f).returncode != 0:
print('Training failed')
print(f.read())
raise Exception()
with open(workspace_spec_json, 'r') as f:
self.config[DEFAULT_SECTION][WORKSPACE_ID_ITEM] = json.load(
f)[WORKSPACE_ID_TAG]
self.config[DEFAULT_SECTION][MODE_ITEM] = KFOLD
self.config[DEFAULT_SECTION][TEMP_DIR_ITEM] = kfold_test_dir
self.config[DEFAULT_SECTION][FIGURE_PATH_ITEM] = \
os.path.join(kfold_test_dir, 'figure.png')
self.config[DEFAULT_SECTION][FOLD_NUM_ITEM] = str(FOLD_NUM_DEFAULT)
self.config[DEFAULT_SECTION][PARTIAL_CREDIT_TABLE_ITEM] = \
os.path.join(tool_base, 'resources', 'sample',
'partial-credit-table.csv')
kfold_config_path = os.path.join(kfold_test_dir, 'config.ini')
with open(kfold_config_path, 'w') as configfile:
self.config.write(configfile)
args = [sys.executable, self.script_path, '-c', kfold_config_path]
if subprocess.run(args).returncode != 0:
raised = True
workspace_ids = []
workspace_ids.append(self.config[DEFAULT_SECTION][WORKSPACE_ID_ITEM])
delete_workspaces(apikey,
self.config[WCS_CREDS_SECTION][WCS_BASEURL_ITEM],
'2019-02-28', workspace_ids)
self.assertFalse(raised, 'Exception raised')
def kfold(fold_num, temp_dir, intent_train_file, workspace_base_file,
figure_path, keep_workspace, username, password, weight_mode,
conf_thres):
FOLD_TRAIN = 'fold_train'
FOLD_TEST = 'fold_test'
WORKSPACE_SPEC = 'fold_workspace'
WORKSPACE_NAME = 'workspace_name'
TEST_OUT = 'test_out'
print('Begin {} with following details:'.format(KFOLD.upper()))
print('{}={}'.format(INTENT_FILE_ITEM, intent_train_file))
print('{}={}'.format(WORKSPACE_BASE_ITEM, workspace_base_file))
print('{}={}'.format(FIGURE_PATH_ITEM, figure_path))
print('{}={}'.format(TEMP_DIR_ITEM, temp_dir))
print('{}={}'.format(FOLD_NUM_ITEM, fold_num))
print('{}={}'.format(DO_KEEP_WORKSPACE_ITEM, BOOL_MAP[keep_workspace]))
print('{}={}'.format(WEIGHT_MODE_ITEM, weight_mode))
print('{}={}'.format(CONF_THRES_ITEM, conf_thres))
print('{}={}'.format(WCS_USERNAME_ITEM, username))
working_dir = os.path.join(temp_dir, KFOLD)
if not os.path.exists(working_dir):
os.makedirs(working_dir)
# Prepare folds
if subprocess.run([
sys.executable, CREATE_TEST_TRAIN_FOLDS_PATH, '-i',
intent_train_file, '-o', working_dir, '-k',
str(fold_num)
],
stdout=subprocess.PIPE).returncode == 0:
print('Created {} folds'.format(str(fold_num)))
else:
raise RuntimeError('Failure in folds creation')
# Construct fold params
fold_params = [{
FOLD_TRAIN:
os.path.join(working_dir, str(idx), TRAIN_FILENAME),
FOLD_TEST:
os.path.join(working_dir, str(idx), TEST_FILENAME),
TEST_OUT:
os.path.join(working_dir, str(idx), TEST_OUT_FILENAME),
WORKSPACE_SPEC:
os.path.join(working_dir, str(idx), SPEC_FILENAME),
WORKSPACE_NAME:
'{}_{}'.format(KFOLD, str(idx))
} for idx in range(fold_num)]
# Begin training
train_processes_specs = {}
for fold_param in fold_params:
spec_file = open(fold_param[WORKSPACE_SPEC], 'w')
train_args = [
sys.executable, TRAIN_CONVERSATION_PATH, '-i',
fold_param[FOLD_TRAIN], '-n', fold_param[WORKSPACE_NAME], '-u',
username, '-p', password, '-w', workspace_base_file
]
train_processes_specs[subprocess.Popen(train_args,
stdout=spec_file)] = spec_file
train_failure_idx = []
for idx, (process, file) in enumerate(train_processes_specs.items()):
if process.wait() == 0:
file.close()
else:
train_failure_idx.append(idx)
try:
if len(train_failure_idx) != 0:
raise RuntimeError('Fail to train {} fold workspace'.format(
','.join(str(train_failure_idx))))
print('Trained {} workspaces'.format(str(fold_num)))
# Begin testing
test_processes = []
workspace_ids = []
FOLD_TEST_RATE = int(MAX_TEST_RATE / fold_num)
for fold_param in fold_params:
workspace_id = None
with open(fold_param[WORKSPACE_SPEC]) as f:
workspace_id = json.load(f)[WORKSPACE_ID_TAG]
workspace_ids.append(workspace_id)
test_args = [
sys.executable, TEST_CONVERSATION_PATH, '-i',
fold_param[FOLD_TEST], '-o', fold_param[TEST_OUT], '-u',
username, '-p', password, '-t', UTTERANCE_COLUMN, '-g',
GOLDEN_INTENT_COLUMN, '-w', workspace_id, '-r',
str(FOLD_TEST_RATE), '-m'
]
test_processes.append(subprocess.Popen(test_args))
test_failure_idx_str = []
for idx, process in enumerate(test_processes):
if process.wait() != 0:
test_failure_idx_str.append(str(idx))
if len(test_failure_idx_str) != 0:
raise RuntimeError('Fail to test {} fold workspace'.format(
','.join(test_failure_idx_str)))
print('Tested {} workspaces'.format(str(fold_num)))
test_out_files = [fold_param[TEST_OUT] for fold_param in fold_params]
# Union test out
pd.concat([pd.read_csv(file, quoting=csv.QUOTE_ALL, encoding=UTF_8,
keep_default_na=False)
for file in test_out_files]) \
.to_csv(os.path.join(working_dir, KFOLD_UNION_FILE),
encoding='utf-8', quoting=csv.QUOTE_ALL, index=False)
classfier_names = ['Fold {}'.format(idx) for idx in range(fold_num)]
plot_args = [sys.executable, CREATE_PRECISION_CURVE_PATH,
'-t', '{} Fold Test'.format(str(fold_num)),
'-o', figure_path, '-w', weight_mode,
'--tau', conf_thres, '-n'] + \
classfier_names + ['-i'] + test_out_files
if subprocess.run(plot_args).returncode == 0:
print('Generated precision curves for {} folds'.format(
str(fold_num)))
else:
raise RuntimeError('Failure in plotting curves')
finally:
if not keep_workspace:
workspace_ids = []
for idx in range(fold_num):
if idx not in train_failure_idx:
with open(fold_params[idx][WORKSPACE_SPEC]) as f:
workspace_id = json.load(f)[WORKSPACE_ID_TAG]
workspace_ids.append(workspace_id)
delete_workspaces(username, password, workspace_ids)
def blind(temp_dir, intent_train_file, workspace_base_file, figure_path,
test_out_path, test_input_file, previous_blind_out, keep_workspace,
username, password, weight_mode, conf_thres):
print('Begin {} with following details:'.format(BLIND_TEST.upper()))
print('{}={}'.format(INTENT_FILE_ITEM, intent_train_file))
print('{}={}'.format(WORKSPACE_BASE_ITEM, workspace_base_file))
print('{}={}'.format(TEST_FILE_ITEM, test_input_file))
print('{}={}'.format(PREVIOUS_BLIND_OUT_ITEM, previous_blind_out))
print('{}={}'.format(FIGURE_PATH_ITEM, figure_path))
print('{}={}'.format(TEST_OUT_PATH_ITEM, test_out_path))
print('{}={}'.format(TEMP_DIR_ITEM, temp_dir))
print('{}={}'.format(DO_KEEP_WORKSPACE_ITEM, BOOL_MAP[keep_workspace]))
print('{}={}'.format(WEIGHT_MODE_ITEM, weight_mode))
print('{}={}'.format(CONF_THRES_ITEM, conf_thres))
print('{}={}'.format(WCS_USERNAME_ITEM, username))
# Validate previous blind out format
test_out_files = [test_out_path]
classfier_names = ['New Classifier']
if previous_blind_out is not None:
df = pd.read_csv(previous_blind_out,
quoting=csv.QUOTE_ALL,
encoding=UTF_8)
header = list(df.columns.values)
if INTENT_JUDGE_COLUMN not in header:
raise ValueError("'{}' column doesn't exist in {}.".format(
INTENT_JUDGE_COLUMN, previous_blind_out))
test_out_files.append(previous_blind_out)
classfier_names.append('Old Classifier')
working_dir = os.path.join(temp_dir, BLIND_TEST)
if not os.path.exists(working_dir):
os.makedirs(working_dir)
workspace_spec_json = os.path.join(working_dir, SPEC_FILENAME)
train_args = [
sys.executable, TRAIN_CONVERSATION_PATH, '-i', intent_train_file, '-n',
'blind test', '-u', username, '-p', password, '-w', workspace_base_file
]
with open(workspace_spec_json, 'w') as f:
if subprocess.run(train_args, stdout=f).returncode == 0:
print('Trained blind workspace')
else:
raise RuntimeError('Failure in training workspace')
workspace_id = None
with open(workspace_spec_json, 'r') as f:
workspace_id = json.load(f)[WORKSPACE_ID_TAG]
try:
if subprocess.run([
sys.executable, TEST_CONVERSATION_PATH, '-i', test_input_file,
'-o', test_out_path, '-m', '-u', username, '-p', password,
'-t', UTTERANCE_COLUMN, '-g', GOLDEN_INTENT_COLUMN, '-w',
workspace_id, '-r',
str(MAX_TEST_RATE)
]).returncode == 0:
print('Tested blind workspace')
else:
raise RuntimeError('Failure in testing blind data')
if subprocess.run([
sys.executable, CREATE_PRECISION_CURVE_PATH, '-t',
'Golden Test Set', '-w', weight_mode, '--tau', conf_thres,
'-o', figure_path, '-n'
] + classfier_names + ['-i'] + test_out_files).returncode == 0:
print('Generated precision curves for blind set')
else:
raise RuntimeError('Failure in plotting curves')
finally:
if not keep_workspace:
delete_workspaces(username, password, [workspace_id])
