def create_shuffle_stratified_splits(dataset,
n_splits,
train_size_ratio=1.0,
drop_entities=False,
seed=None):
if train_size_ratio > 1.0 or train_size_ratio < 0:
raise ValueError("Invalid value for train size ratio: %s" %
train_size_ratio)
nb_utterances = {
intent: len(data[UTTERANCES])
for intent, data in dataset[INTENTS].items()
}
total_utterances = sum(nb_utterances.values())
if total_utterances < n_splits:
raise NotEnoughDataError("Number of utterances is too low (%s)" %
total_utterances)
if drop_entities:
dataset = deepcopy(dataset)
for entity, data in iteritems(dataset[ENTITIES]):
data[DATA] = []
else:
dataset = update_entities_with_utterances(dataset)
utterances = np.array([
(intent_name, utterance)
for intent_name, intent_data in dataset[INTENTS].items()
for utterance in intent_data[UTTERANCES]
])
intents = np.array([u[0] for u in utterances])
X = np.zeros(len(intents))
random_state = check_random_state(seed)
sss = StratifiedKFold(n_splits=n_splits,
shuffle=True,
random_state=random_state)
splits = []
try:
for train_index, test_index in sss.split(X, intents):
train_utterances = utterances[train_index].tolist()
train_utterances = get_utterances_subset(train_utterances,
train_size_ratio)
test_utterances = utterances[test_index].tolist()
if len(train_utterances) == 0:
not_enough_data(n_splits, train_size_ratio)
train_dataset = deepcopy(dataset)
train_dataset[INTENTS] = dict()
for intent_name, utterance in train_utterances:
if intent_name not in train_dataset[INTENTS]:
train_dataset[INTENTS][intent_name] = {UTTERANCES: []}
train_dataset[INTENTS][intent_name][UTTERANCES].append(
deepcopy(utterance))
splits.append((train_dataset, test_utterances))
except ValueError:
not_enough_data(n_splits, train_size_ratio)
return splits
def test_get_utterances_subset_should_work(self):
# Given
utterances = [
("intent1", {
"data": [{
"text": "text1"
}]
}),
("intent1", {
"data": [{
"text": "text2"
}]
}),
("intent1", {
"data": [{
"text": "text3"
}]
}),
("intent1", {
"data": [{
"text": "text4"
}]
}),
("intent2", {
"data": [{
"text": "text1"
}]
}),
("intent2", {
"data": [{
"text": "text2"
}]
}),
("intent3", {
"data": [{
"text": "text1"
}]
}),
("intent3", {
"data": [{
"text": "text2"
}]
}),
("intent3", {
"data": [{
"text": "text3"
}]
}),
("intent3", {
"data": [{
"text": "text4"
}]
}),
("intent3", {
"data": [{
"text": "text5"
}]
}),
("intent3", {
"data": [{
"text": "text6"
}]
}),
]
# When
utterances_subset = get_utterances_subset(utterances, ratio=0.5)
utterances_subset = sorted(utterances_subset,
key=lambda u: "%s%s" %
(u[0], u[1]["data"][0]["text"]))
# Then
expected_utterances = [
("intent1", {
"data": [{
"text": "text1"
}]
}),
("intent1", {
"data": [{
"text": "text2"
}]
}),
("intent2", {
"data": [{
"text": "text1"
}]
}),
("intent3", {
"data": [{
"text": "text1"
}]
}),
("intent3", {
"data": [{
"text": "text2"
}]
}),
("intent3", {
"data": [{
"text": "text3"
}]
}),
]
self.assertListEqual(expected_utterances, utterances_subset)
def create_shuffle_stratified_splits(
dataset,
n_splits,
train_size_ratio=1.0,
drop_entities=False,
seed=None,
out_of_domain_utterances=None,
intents_filter=None,
):
if train_size_ratio > 1.0 or train_size_ratio < 0:
error_msg = "Invalid value for train size ratio: %s" % train_size_ratio
logger.exception(error_msg)
raise ValueError(error_msg)
nb_utterances = {
intent: len(data[UTTERANCES])
for intent, data in dataset[INTENTS].items()
}
if any(
(nb * train_size_ratio < n_splits for nb in nb_utterances.values())):
raise NotEnoughDataError(dataset, n_splits, train_size_ratio)
if drop_entities:
dataset = deepcopy(dataset)
for entity, data in dataset[ENTITIES].items():
data[DATA] = []
else:
dataset = update_entities_with_utterances(dataset)
utterances = np.array([
(intent_name, utterance)
for intent_name, intent_data in dataset[INTENTS].items()
for utterance in intent_data[UTTERANCES]
])
intents = np.array([u[0] for u in utterances])
X = np.zeros(len(intents))
random_state = check_random_state(seed)
sss = StratifiedKFold(n_splits=n_splits,
shuffle=True,
random_state=random_state)
splits = []
for train_index, test_index in sss.split(X, intents):
train_utterances = utterances[train_index].tolist()
train_utterances = get_utterances_subset(train_utterances,
train_size_ratio)
test_utterances = utterances[test_index].tolist()
train_dataset = deepcopy(dataset)
train_dataset[INTENTS] = dict()
for intent_name, utterance in train_utterances:
if intent_name not in train_dataset[INTENTS]:
train_dataset[INTENTS][intent_name] = {UTTERANCES: []}
train_dataset[INTENTS][intent_name][UTTERANCES].append(
deepcopy(utterance))
splits.append((train_dataset, test_utterances))
if intents_filter is not None:
filtered_splits = []
for train_dataset, test_utterances in splits:
test_utterances = [(intent_name, utterance)
for intent_name, utterance in test_utterances
if intent_name in intents_filter]
filtered_splits.append((train_dataset, test_utterances))
splits = filtered_splits
if out_of_domain_utterances is not None:
additional_test_utterances = [[
NONE_INTENT_NAME, {
DATA: [{
TEXT: utterance
}]
}
] for utterance in out_of_domain_utterances]
for split in splits:
split[1].extend(additional_test_utterances)
return splits