def test_random_policy(self):
policy = RandomPolicy(2, sequence_length=2)
n_samples = 100
samples = [policy.generate() for _ in range(n_samples)]
a_ct = samples.count([0, 0])
b_ct = samples.count([0, 1])
c_ct = samples.count([1, 0])
d_ct = samples.count([1, 1])
self.assertGreater(a_ct, 0)
self.assertGreater(b_ct, 0)
self.assertGreater(c_ct, 0)
self.assertGreater(d_ct, 0)
self.assertEqual(a_ct + b_ct + c_ct + d_ct, n_samples)
def test_tf_applier_pandas_generator(self):
df = self._get_x_df_with_str()
policy = RandomPolicy(1,
sequence_length=2,
n_per_original=2,
keep_original=False)
applier = PandasTFApplier([square], policy)
gen = applier.apply_generator(df, batch_size=2)
df_expected = [
pd.DataFrame(
{
"num": [1, 1, 16, 16],
"strs": ["x", "x", "y", "y"]
},
index=[0, 0, 1, 1],
),
pd.DataFrame({
"num": [81, 81],
"strs": ["z", "z"]
}, index=[2, 2]),
]
for df_batch, df_batch_expected in zip(gen, df_expected):
self.assertEqual(df_batch.num.dtype, "int64")
pd.testing.assert_frame_equal(df_batch, df_batch_expected)
pd.testing.assert_frame_equal(df, self._get_x_df_with_str())
def main():
df = pd.read_csv('../airbnb/reviews.tsv', sep='\t')
newdf = df[['comments', 'Great (1) Not Great (0)']]
newdf.columns = ['text', 'label']
chunks = []
labels = []
buffer = []
for i, row in newdf.iterrows():
sents = nltk.sent_tokenize(row['text'])
for sent in sents:
buffer.append(sent)
if (len(buffer)) % 3 == 0:
chunks.append(" ".join(buffer))
labels.append(row['label'])
buffer = [buffer[random.randint(0, 2)]]
if len(buffer) > 1:
chunks.append(" ".join(buffer))
labels.append(row['label'])
buffer = []
chunkedDf = pd.DataFrame({'text': chunks, 'label': labels})
random_policy = RandomPolicy(len(tfs),
sequence_length=4,
n_per_original=1,
keep_original=True)
tf_applier = PandasTFApplier(tfs, random_policy)
newdf_augmented = tf_applier.apply(chunkedDf)
print(len(newdf))
print(len(newdf_augmented))
newdf_augmented.to_csv('airbnb_augmented.csv')
def test_tf_applier_returns_none(self) -> None:
data = self._get_x_namespace()
policy = RandomPolicy(
1, sequence_length=2, n_per_original=2, keep_original=True
)
applier = TFApplier([square_returns_none], policy)
data_augmented = applier.apply(data, progress_bar=False)
vals = [1, 1, 1, 2, 3, 81, 81]
self.assertEqual(data_augmented, self._get_x_namespace(vals))
self.assertEqual(data, self._get_x_namespace())
def test_tf_applier_returns_none_generator(self) -> None:
data = self._get_x_namespace()
policy = RandomPolicy(
1, sequence_length=2, n_per_original=2, keep_original=True
)
applier = TFApplier([square_returns_none], policy)
batches_expected = [[1, 1, 1, 2], [3, 81, 81]]
gen = applier.apply_generator(data, batch_size=2)
for batch, batch_expected in zip(gen, batches_expected):
self.assertEqual(batch, self._get_x_namespace(batch_expected))
self.assertEqual(data, self._get_x_namespace())
def test_tf_applier_returns_none(self):
df = self._get_x_df()
policy = RandomPolicy(1,
sequence_length=2,
n_per_original=2,
keep_original=True)
applier = PandasTFApplier([square_returns_none], policy)
df_augmented = applier.apply(df, progress_bar=False)
df_expected = pd.DataFrame(dict(num=[1, 1, 1, 2, 3, 81, 81]),
index=[0, 0, 0, 1, 2, 2, 2])
self.assertEqual(df_augmented.num.dtype, "int64")
pd.testing.assert_frame_equal(df_augmented, df_expected)
pd.testing.assert_frame_equal(df, self._get_x_df())
def test_tf_applier(self) -> None:
data = self._get_x_namespace()
policy = RandomPolicy(
1, sequence_length=2, n_per_original=1, keep_original=False
)
applier = TFApplier([square], policy)
data_augmented = applier.apply(data, progress_bar=False)
self.assertEqual(data_augmented, self._get_x_namespace([1, 16, 81]))
self.assertEqual(data, self._get_x_namespace())
data_augmented = applier.apply(data, progress_bar=True)
self.assertEqual(data_augmented, self._get_x_namespace([1, 16, 81]))
self.assertEqual(data, self._get_x_namespace())
def test_tf_applier_returns_none_generator(self):
df = self._get_x_df()
policy = RandomPolicy(
1, sequence_length=2, n_per_original=2, keep_original=True
)
applier = PandasTFApplier([square_returns_none], policy)
gen = applier.apply_generator(df, batch_size=2)
df_expected = [
make_df([1, 1, 1, 2], [0, 0, 0, 1]),
make_df([3, 81, 81], [2, 2, 2]),
]
for df_batch, df_batch_expected in zip(gen, df_expected):
pd.testing.assert_frame_equal(df_batch, df_batch_expected)
pd.testing.assert_frame_equal(df, self._get_x_df())
def apply_tf_on_data(df_train):
"""
Here we apply the transformation functions (from transformation_function.py)
on the given train data frame.
Return the enlarged data frame.
"""
print("")
print("Transformation Functions:")
tfs = [transformation_function.change_perek, transformation_function.change_masechet]
random_policy = RandomPolicy(
len(tfs), sequence_length=len(tfs), n_per_original=TRANSFORMATION_FACTOR, keep_original=True
)
print("-Applying ["+str(len(tfs))+"] transformation functions with factor ["+str(TRANSFORMATION_FACTOR)+"] ...")
tf_applier = PandasTFApplier(tfs, random_policy)
df_train_augmented = tf_applier.apply(df_train)
# Y_train_augmented = df_train_augmented["tag"].values
print("DONE")
return df_train_augmented
def test_tf_applier_pandas(self):
df = self._get_x_df_with_str()
policy = RandomPolicy(1,
sequence_length=2,
n_per_original=1,
keep_original=False)
applier = PandasTFApplier([square], policy)
df_augmented = applier.apply(df, progress_bar=False)
df_expected = pd.DataFrame(dict(num=[1, 16, 81], strs=STR_DATA),
index=[0, 1, 2])
self.assertEqual(df_augmented.num.dtype, "int64")
pd.testing.assert_frame_equal(df_augmented, df_expected)
pd.testing.assert_frame_equal(df, self._get_x_df_with_str())
df_augmented = applier.apply(df, progress_bar=True)
df_expected = pd.DataFrame(dict(num=[1, 16, 81], strs=STR_DATA),
index=[0, 1, 2])
pd.testing.assert_frame_equal(df_augmented, df_expected)
pd.testing.assert_frame_equal(df, self._get_x_df_with_str())
def augmentation_evaluation(df_train, df_test, policy, p=None):
tfs = [
TransformationFunction.change_addr,
TransformationFunction.change_business,
TransformationFunction.change_o,
TransformationFunction.randomly_delete,
TransformationFunction.randomly_add
]
if policy == "random":
random_policy = RandomPolicy(len(tfs),
sequence_length=2,
n_per_original=2,
keep_original=True)
tf_applier = PandasTFApplier(tfs, random_policy)
df_train_augmented = tf_applier.apply(df_train)
Y_train_augmented = df_train_augmented["label"].values
print(f"Original training set size: {len(df_train)}")
print(f"Augmented training set size: {len(df_train_augmented)}")
return df_train_augmented, Y_train_augmented
if policy == "mean":
if p is None:
p = [0.1, 0.1, 0.1, 0.35, 0.35]
mean_field_policy = MeanFieldPolicy(
len(tfs),
sequence_length=
2, # how many TFs to apply uniformly at random per data point
n_per_original=
2, # how many augmented data points to generate per original data point
keep_original=True,
p=p, # specify a sampling distribution for the TFs
)
tf_applier = PandasTFApplier(tfs, mean_field_policy)
df_train_augmented = tf_applier.apply(df_train)
Y_train_augmented = df_train_augmented["label"].values
print(f"Original training set size: {len(df_train)}")
print(f"Augmented training set size: {len(df_train_augmented)}")
return df_train_augmented, Y_train_augmented
# As we'll see below, Snorkel is compatible with such learned augmentation policies.
# %% [markdown]
# ## 3. Applying Transformation Functions
# %% [markdown]
# We'll first define a `Policy` to determine what sequence of TFs to apply to each data point.
# We'll start with a [`RandomPolicy`](https://snorkel.readthedocs.io/en/master/packages/_autosummary/augmentation/snorkel.augmentation.RandomPolicy.html)
# that samples `sequence_length=2` TFs to apply uniformly at random per data point.
# The `n_per_original` argument determines how many augmented data points to generate per original data point.
# %%
from snorkel.augmentation import RandomPolicy
random_policy = RandomPolicy(len(tfs),
sequence_length=2,
n_per_original=2,
keep_original=True)
# %% [markdown]
# In some cases, we can do better than uniform random sampling.
# We might have domain knowledge that some TFs should be applied more frequently than others,
# or have trained an [automated data augmentation model](https://snorkel.org/tanda/)
# that learned a sampling distribution for the TFs.
# Snorkel supports this use case with a
# [`MeanFieldPolicy`](https://snorkel.readthedocs.io/en/master/packages/_autosummary/augmentation/snorkel.augmentation.MeanFieldPolicy.html),
# which allows you to specify a sampling distribution for the TFs.
# We give higher probabilities to the `replace_[X]_with_synonym` TFs, since those provide more information to the model.
# %%
from snorkel.augmentation import MeanFieldPolicy