def test_filter_should_give_Transformable_rdd(self):
initial_dataset = self.sc.parallelize(["1,2", "1,2", "1,3"])
transformable_rdd = TransformableRDD(initial_dataset, "csv")
rdd_filter = transformable_rdd.filter(
lambda line: line.split(",")[1] != "2")
collected = rdd_filter.collect()
self.assertEqual(1, collected.__len__())
def test_should_smooth_data_by_Simple_Moving_Average(self):
initial_dataset = self.sc.parallelize(
["52,3,53", "23,4,64", "23,5,64", "23,6,64", "23,7,64", "23,8,64", "23,9,64"], 3)
transformable_rdd = TransformableRDD(initial_dataset, "csv")
transformed = transformable_rdd.smooth(1, SimpleMovingAverage(3))
excepted = 4.0
self.assertEquals(excepted, transformed.first())
def test_pivot_table_by_count_should_give_pivoted_table(self):
initial_dataSet = self.sc.parallelize([
"known,new,long,home,skips",
"unknown,new,short,work,reads",
"unknown,follow Up,long,work,skips",
"known,follow Up,long,home,skips",
"known,new,short,home,reads",
"known,follow Up,long,work,skips",
"unknown,follow Up,short,work,skips",
"unknown,new,short,work,reads",
"known,follow Up,long,home,skips",
"known,new,long,work,skips",
"unknown,follow Up,short,home,skips",
"known,new,long,work,skips",
"known,follow Up,short,home,reads",
"known,new,short,work,reads",
"known,new,short,home,reads",
"known,follow Up,short,work,reads",
"known,new,short,home,reads",
"unknown,new,short,work,reads"
])
initial_rdd = TransformableRDD(initial_dataSet, "csv")
table = initial_rdd.pivot_by_count(4, [0, 1, 2, 3])
entry = table.value_at("skips", "known")
self.assertEqual(6, entry)
self.assertEqual(3, table.value_at("skips", "unknown"))
def test_should_split_given_column_indexes_split_by_delimiter_with_retain_column(self):
initial_data_set = self.sc.parallelize(["FirstName LastName MiddleName,850"])
initial_rdd = TransformableRDD(initial_data_set, "csv")
split_with_retained_columns = initial_rdd.split_by_delimiter(0, " ", True)
self.assertEquals("FirstName LastName MiddleName,850,FirstName,LastName,MiddleName",
split_with_retained_columns.first())
def test_replace_values_should_replace_cluster_values_with_given_text(self):
initial_dataset = self.sc.parallelize(["XA,Y", "A,B", "AX,Z", "A,Q", "A,E"])
transformable_rdd = TransformableRDD(initial_dataset)
clusters = transformable_rdd.clusters(0, NGramFingerprintAlgorithm(1))
one_cluster = clusters.get_all_clusters()[0]
values = transformable_rdd.replace_values(one_cluster, "Hello", 0).collect()
self.assertTrue(values.__contains__("Hello,B"))
def test_to_double_rdd_should_change_string_to_double_rdd(self):
initial_dataset = self.sc.parallelize(["1,1", "5,2", "8,3"])
transformable_rdd = TransformableRDD(initial_dataset)
rdd = transformable_rdd.to_double_rdd(0)
collected = rdd.collect()
self.assertTrue(collected.__contains__(1.0))
self.assertTrue(collected.__contains__(5.0))
self.assertTrue(collected.__contains__(8.0))
def test_multiply_column_should_multiply_two_given_column(self):
initial_dataset = self.sc.parallelize(["1,1", "1,2", "1,3"])
transformable_rdd = TransformableRDD(initial_dataset)
multiplied_rdd = transformable_rdd.multiply_columns(0, 1)
collected = multiplied_rdd.collect()
self.assertTrue(collected.__contains__(1.0))
self.assertTrue(collected.__contains__(2.0))
self.assertTrue(collected.__contains__(3.0))
def test_clusters_should_give_clusters_of_given_column_index(self):
rdd = self.sc.parallelize(["CLUSTER Of Finger print", "finger print of cluster", "finger print for cluster"])
transformable_rdd = TransformableRDD(rdd, 'csv')
clusters = transformable_rdd.clusters(0, SimpleFingerprint())
list_of_clusters = clusters.get_all_clusters()
one_cluster = list_of_clusters[0]
self.assertTrue(one_cluster.__contains__("CLUSTER Of Finger print"))
self.assertFalse(one_cluster.__contains__("finger print for cluster"))
def test_clusters_should_give_clusters_By_n_gram_fingerprint(self):
rdd = self.sc.parallelize(["CLUSTER Of Finger print", "finger print of cluster", "finger print for cluster"])
transformable_rdd = TransformableRDD(rdd, 'csv')
clusters = transformable_rdd.clusters(0, NGramFingerprintAlgorithm(1))
list_of_clusters = clusters.get_all_clusters()
one_cluster = list_of_clusters[0]
self.assertTrue(one_cluster.__contains__("CLUSTER Of Finger print"))
self.assertTrue(one_cluster.__contains__("finger print for cluster"))
def test_to_double_rdd_should_change_string_to_double_rdd(self):
initial_dataset = self.sc.parallelize(["1,1", "5,2", "8,3"])
transformable_rdd = TransformableRDD(initial_dataset)
rdd = transformable_rdd.to_double_rdd(0)
collected = rdd.collect()
self.assertTrue(collected.__contains__(1.0))
self.assertTrue(collected.__contains__(5.0))
self.assertTrue(collected.__contains__(8.0))
def test_transformableRDD_can_impute_the_missing_values_by_UnivariateLinearRegressionSubstitution(
self):
rdd = self.sc.parallelize(
["60,3.1", "61,3.6", "62,3.8", "63,4", "65,4.1", "64,"])
transformable_rdd = TransformableRDD(rdd, 'csv')
imputed_rdd = transformable_rdd.impute(
1, UnivariateLinearRegressionSubstitution(0))
self.assertTrue(imputed_rdd.collect().__contains__("64,4.06"))
def test_multiply_column_should_multiply_two_given_column(self):
initial_dataset = self.sc.parallelize(["1,1", "1,2", "1,3"])
transformable_rdd = TransformableRDD(initial_dataset)
multiplied_rdd = transformable_rdd.multiply_columns(0, 1)
collected = multiplied_rdd.collect()
self.assertTrue(collected.__contains__(1.0))
self.assertTrue(collected.__contains__(2.0))
self.assertTrue(collected.__contains__(3.0))
def test_should_split_given_column_indexes_split_by_delimiter(self):
initial_data_set = self.sc.parallelize(
["FirstName LastName MiddleName,850"])
initial_rdd = TransformableRDD(initial_data_set, "csv")
splitted_columns = initial_rdd.split_by_delimiter(0, " ", False)
self.assertEquals("850,FirstName,LastName,MiddleName",
splitted_columns.first())
def test_should_smooth_data_by_Simple_Moving_Average(self):
initial_dataset = self.sc.parallelize([
"52,3,53", "23,4,64", "23,5,64", "23,6,64", "23,7,64", "23,8,64",
"23,9,64"
], 3)
transformable_rdd = TransformableRDD(initial_dataset, "csv")
transformed = transformable_rdd.smooth(1, SimpleMovingAverage(3))
excepted = 4.0
self.assertEquals(excepted, transformed.first())
def test_map_should_give_Transformable_rdd(self):
initial_dataset = self.sc.parallelize(["1,2", "1,2", "1,3"])
transformable_rdd = TransformableRDD(initial_dataset, "csv")
rdd_map = transformable_rdd.map(lambda line: line + "yes")
deduplicate = rdd_map.deduplicate()
collected = deduplicate.collect()
self.assertEqual(2, collected.__len__())
expected = "1,2yes"
self.assertTrue(collected.__contains__(expected))
def test_map_should_give_Transformable_rdd(self):
initial_dataset = self.sc.parallelize(["1,2", "1,2", "1,3"])
transformable_rdd = TransformableRDD(initial_dataset, "csv")
rdd_map = transformable_rdd.map(lambda line: line + "yes")
deduplicate = rdd_map.deduplicate()
collected = deduplicate.collect()
self.assertEqual(2, collected.__len__())
expected = "1,2yes"
self.assertTrue(collected.__contains__(expected))
def test_should_split_given_column_by_field_length_with_retained_columns(self):
data = ["John,Male,21,+914382313832,Canada", "Smith, Male, 30,+015314343462, UK",
"Larry, Male, 23,+009815432975, USA", "Fiona, Female,18,+891015709854,USA"]
initial_data_set = self.sc.parallelize(data)
initial_rdd = TransformableRDD(initial_data_set, "csv")
result = initial_rdd.split_by_field_length(3, [3, 10], True).collect()
self.assertTrue(len(result) == 4)
self.assertTrue(result.__contains__("John,Male,21,+914382313832,Canada,+91,4382313832"))
self.assertTrue(result.__contains__("Smith,Male,30,+015314343462,UK,+01,5314343462"))
def test_should_merge_given_column_indexes(self):
initial_data_set = self.sc.parallelize(["FirstName,LastName,732,MiddleName"])
initial_rdd = TransformableRDD(initial_data_set, "csv")
joined_column_rdd = initial_rdd.merge_columns([3, 1, 0], "_", False)
self.assertEquals("732,MiddleName_LastName_FirstName", joined_column_rdd.first())
with_originals = initial_rdd.merge_columns([3, 1, 0], "_", True)
self.assertEquals("FirstName,LastName,732,MiddleName,MiddleName_LastName_FirstName", with_originals.first())
joined_column_with_defaults = initial_rdd.merge_columns([3, 1, 0])
self.assertEquals("732,MiddleName LastName FirstName", joined_column_with_defaults.first())
def test_transformableRDD_can_impute_the_missing_values_by_NaiveBayesSubstitution(self):
rdd = self.sc.parallelize(["Drew,No,Blue,Short,Male",
"Claudia,Yes,Brown,Long,Female",
"Drew,No,Blue,Long,Female",
"Drew,No,Blue,Long,Female",
"Alberto,Yes,Brown,Short,Male",
"Karin,No,Blue,Long,Female",
"Nina,Yes,Brown,Short,Female",
"Sergio,Yes,Blue,Long,Male",
"Drew,Yes,Blue,Long,"])
transformable_rdd = TransformableRDD(rdd, 'csv')
imputed_rdd = transformable_rdd.impute(4, NaiveBayesSubstitution(0, 1, 2, 3))
self.assertTrue(imputed_rdd.collect().__contains__("Drew,Yes,Blue,Long,Female"))
def test_transformableRDD_can_impute_the_missing_values_by_NaiveBayesSubstitution(
self):
rdd = self.sc.parallelize([
"Drew,No,Blue,Short,Male", "Claudia,Yes,Brown,Long,Female",
"Drew,No,Blue,Long,Female", "Drew,No,Blue,Long,Female",
"Alberto,Yes,Brown,Short,Male", "Karin,No,Blue,Long,Female",
"Nina,Yes,Brown,Short,Female", "Sergio,Yes,Blue,Long,Male",
"Drew,Yes,Blue,Long,"
])
transformable_rdd = TransformableRDD(rdd, 'csv')
imputed_rdd = transformable_rdd.impute(
4, NaiveBayesSubstitution(0, 1, 2, 3))
self.assertTrue(
imputed_rdd.collect().__contains__("Drew,Yes,Blue,Long,Female"))
def test_should_split_the_given_column_by_delimiter_into_given_number_of_split(self):
data = [
"John\tMale\t21\t+91-4382-313832\tCanada",
"Smith\tMale\t30\t+01-5314-343462\tUK",
"Larry\tMale\t23\t+00-9815-432975\tUSA",
"Fiona\tFemale\t18\t+89-1015-709854\tUSA"
]
initial_data_set = self.sc.parallelize(data)
initial_rdd = TransformableRDD(initial_data_set, "tsv")
new_dataset = initial_rdd.split_by_delimiter(3, "-", False, 2)
list_of_records = new_dataset.collect()
self.assertEqual(4, list_of_records.__len__())
self.assertTrue(list_of_records.__contains__("John\tMale\t21\tCanada\t+91\t4382-313832"))
self.assertTrue(list_of_records.__contains__("Smith\tMale\t30\tUK\t+01\t5314-343462"))
def test_should_smooth_data_by_Weighted_Moving_Average(self):
initial_dataset = self.sc.parallelize(["10", "12", "16", "13", "17", "19", "15", "20", "22", "19", "21", "19"],
3)
transformable_rdd = TransformableRDD(initial_dataset, "csv")
weights = Weights(3)
weights.add(0.166)
weights.add(0.333)
weights.add(0.5)
moving_average = WeightedMovingAverage(3, weights)
rdd = transformable_rdd.smooth(0, moving_average)
expected = 13.656
actual = rdd.first()
self.assertEquals(expected, actual)
def test_should_split_given_column_by_field_length(self):
data = [
"John,Male,21,+914382313832,Canada",
"Smith, Male, 30,+015314343462, UK",
"Larry, Male, 23,+009815432975, USA",
"Fiona, Female,18,+891015709854,USA"
]
initial_data_set = self.sc.parallelize(data)
initial_rdd = TransformableRDD(initial_data_set, "csv")
result = initial_rdd.split_by_field_length(3, [3, 10], False).collect()
self.assertTrue(len(result) == 4)
self.assertTrue(
result.__contains__("John,Male,21,Canada,+91,4382313832"))
self.assertTrue(result.__contains__("Smith,Male,30,UK,+01,5314343462"))
def test_add_columns_from_should_merge_all_columns_of_other_transformable_rdd(
self):
initial_spelled_numbers = self.sc.parallelize([
"One,Two,Three", "Four,Five,Six", "Seven,Eight,Nine",
"Ten,Eleven,Twelve"
])
spelled_numbers = TransformableRDD(initial_spelled_numbers, "csv")
initial_numeric_data = self.sc.parallelize(
["1\t2\t3", "4\t5\t6", "7\t8\t9", "10\t11\t12"])
numeric_data = TransformableRDD(initial_numeric_data, "tsv")
result = spelled_numbers.add_columns_from(numeric_data).collect()
self.assertTrue(result.__contains__("One,Two,Three,1,2,3"))
self.assertTrue(result.__contains__("Four,Five,Six,4,5,6"))
self.assertTrue(result.__contains__("Seven,Eight,Nine,7,8,9"))
self.assertTrue(result.__contains__("Ten,Eleven,Twelve,10,11,12"))
def test_should_smooth_data_by_Weighted_Moving_Average(self):
initial_dataset = self.sc.parallelize([
"10", "12", "16", "13", "17", "19", "15", "20", "22", "19", "21",
"19"
], 3)
transformable_rdd = TransformableRDD(initial_dataset, "csv")
weights = Weights(3)
weights.add(0.166)
weights.add(0.333)
weights.add(0.5)
moving_average = WeightedMovingAverage(3, weights)
rdd = transformable_rdd.smooth(0, moving_average)
expected = 13.656
actual = rdd.first()
self.assertEquals(expected, actual)
def test_pivot_table_by_count_should_give_pivoted_table(self):
initial_dataSet = self.sc.parallelize([
"known,new,long,home,skips", "unknown,new,short,work,reads",
"unknown,follow Up,long,work,skips",
"known,follow Up,long,home,skips", "known,new,short,home,reads",
"known,follow Up,long,work,skips",
"unknown,follow Up,short,work,skips",
"unknown,new,short,work,reads", "known,follow Up,long,home,skips",
"known,new,long,work,skips", "unknown,follow Up,short,home,skips",
"known,new,long,work,skips", "known,follow Up,short,home,reads",
"known,new,short,work,reads", "known,new,short,home,reads",
"known,follow Up,short,work,reads", "known,new,short,home,reads",
"unknown,new,short,work,reads"
])
initial_rdd = TransformableRDD(initial_dataSet, "csv")
table = initial_rdd.pivot_by_count(4, [0, 1, 2, 3])
entry = table.value_at("skips", "known")
self.assertEqual(6, entry)
self.assertEqual(3, table.value_at("skips", "unknown"))
def test_should_normalize_by_Z_Score_normalization(self):
initial_dataset = self.sc.parallelize([
"07434677419,07371326239,Incoming,211,Wed Sep 15 19:17:44 +0100 2010",
"07641036117,01666472054,Outgoing,0,Mon Feb 11 07:18:23 +0000 1980",
"07641036117,07371326239,Incoming,45,Mon Feb 11 07:45:42 +0000 1980",
"07641036117,07371326239,Incoming,45,Mon Feb 11 07:45:42 +0000 1980",
"07641036117,07681546436,Missed,12,Mon Feb 11 08:04:42 +0000 1980"])
transformable_rdd = TransformableRDD(initial_dataset, 'csv')
final_rdd = transformable_rdd.normalize(3, ZScoreNormalizer())
normalized_durations = final_rdd.select(3).collect()
expected1 = "1.944528306701421"
expected2 = "-0.8202659838241843"
expected3 = "-0.2306179123850742"
expected4 = "-0.2306179123850742"
expected5 = "-0.6630264981070882"
self.assertTrue(normalized_durations.__contains__(expected1))
self.assertTrue(normalized_durations.__contains__(expected2))
self.assertTrue(normalized_durations.__contains__(expected3))
self.assertTrue(normalized_durations.__contains__(expected4))
self.assertTrue(normalized_durations.__contains__(expected5))
def test_should_normalize_by_Min_Max_normalization(self):
initial_dataset = self.sc.parallelize([
"07434677419,07371326239,Incoming,211,Wed Sep 15 19:17:44 +0100 2010",
"07641036117,01666472054,Outgoing,0,Mon Feb 11 07:18:23 +0000 1980",
"07641036117,07371326239,Incoming,45,Mon Feb 11 07:45:42 +0000 1980",
"07641036117,07371326239,Incoming,45,Mon Feb 11 07:45:42 +0000 1980",
"07641036117,07681546436,Missed,12,Mon Feb 11 08:04:42 +0000 1980"])
transformable_rdd = TransformableRDD(initial_dataset, 'csv')
final_rdd = transformable_rdd.normalize(3, MinMaxNormalizer(0, 1))
normalized_durations = final_rdd.select(3).collect()
expected1 = "1.0"
expected2 = "0.0"
expected3 = "0.2132701421800948"
expected4 = "0.2132701421800948"
expected5 = "0.05687203791469194"
self.assertTrue(normalized_durations.__contains__(expected1))
self.assertTrue(normalized_durations.__contains__(expected2))
self.assertTrue(normalized_durations.__contains__(expected3))
self.assertTrue(normalized_durations.__contains__(expected4))
self.assertTrue(normalized_durations.__contains__(expected5))
def test_should_split_the_given_column_by_delimiter_into_given_number_of_split(
self):
data = [
"John\tMale\t21\t+91-4382-313832\tCanada",
"Smith\tMale\t30\t+01-5314-343462\tUK",
"Larry\tMale\t23\t+00-9815-432975\tUSA",
"Fiona\tFemale\t18\t+89-1015-709854\tUSA"
]
initial_data_set = self.sc.parallelize(data)
initial_rdd = TransformableRDD(initial_data_set, "tsv")
new_dataset = initial_rdd.split_by_delimiter(3, "-", False, 2)
list_of_records = new_dataset.collect()
self.assertEqual(4, list_of_records.__len__())
self.assertTrue(
list_of_records.__contains__(
"John\tMale\t21\tCanada\t+91\t4382-313832"))
self.assertTrue(
list_of_records.__contains__(
"Smith\tMale\t30\tUK\t+01\t5314-343462"))
def test_should_normalize_by_Decimal_Scale(self):
initial_dataset = self.sc.parallelize([
"07434677419,07371326239,Incoming,211,Wed Sep 15 19:17:44 +0100 2010",
"07641036117,01666472054,Outgoing,0,Mon Feb 11 07:18:23 +0000 1980",
"07641036117,07371326239,Incoming,45,Mon Feb 11 07:45:42 +0000 1980",
"07641036117,07371326239,Incoming,45,Mon Feb 11 07:45:42 +0000 1980",
"07641036117,07681546436,Missed,12,Mon Feb 11 08:04:42 +0000 1980"])
transformable_rdd = TransformableRDD(initial_dataset, 'csv')
final_rdd = transformable_rdd.normalize(3, DecimalScalingNormalizer())
normalized_durations = final_rdd.select(3).collect()
expected1 = "2.11"
expected2 = "0.0"
expected3 = "0.45"
expected4 = "0.45"
expected5 = "0.12"
self.assertTrue(normalized_durations.__contains__(expected1))
self.assertTrue(normalized_durations.__contains__(expected2))
self.assertTrue(normalized_durations.__contains__(expected3))
self.assertTrue(normalized_durations.__contains__(expected4))
self.assertTrue(normalized_durations.__contains__(expected5))
def test_add_columns_from_should_merge_all_columns_of_other_transformable_rdd(self):
initial_spelled_numbers = self.sc.parallelize([
"One,Two,Three",
"Four,Five,Six",
"Seven,Eight,Nine",
"Ten,Eleven,Twelve"
])
spelled_numbers = TransformableRDD(initial_spelled_numbers, "csv")
initial_numeric_data = self.sc.parallelize([
"1\t2\t3",
"4\t5\t6",
"7\t8\t9",
"10\t11\t12"
])
numeric_data = TransformableRDD(initial_numeric_data, "tsv")
result = spelled_numbers.add_columns_from(numeric_data).collect()
self.assertTrue(result.__contains__("One,Two,Three,1,2,3"))
self.assertTrue(result.__contains__("Four,Five,Six,4,5,6"))
self.assertTrue(result.__contains__("Seven,Eight,Nine,7,8,9"))
self.assertTrue(result.__contains__("Ten,Eleven,Twelve,10,11,12"))
def test_filter_should_give_Transformable_rdd(self):
initial_dataset = self.sc.parallelize(["1,2", "1,2", "1,3"])
transformable_rdd = TransformableRDD(initial_dataset, "csv")
rdd_filter = transformable_rdd.filter(lambda line: line.split(",")[1] != "2")
collected = rdd_filter.collect()
self.assertEqual(1, collected.__len__())
def test_get_duplicates_should_give_duplicates_of_given_column_indexes(
self):
rdd = self.sc.parallelize(["Ram,23", "Ram,23", "Jill,45", "Soa,"])
transformable_rdd = TransformableRDD(rdd, 'csv')
duplicates = transformable_rdd.get_duplicates([0])
self.assertEqual("Ram,23", duplicates.first())
def test_drop_column_should_drop_the_given_column(self):
rdd = self.sc.parallelize(
["Ram,23,Male", "Ram,23,Male", "Jill,45,Female", "Soa,,Female,"])
transformable_rdd = TransformableRDD(rdd, 'csv')
dropped = transformable_rdd.drop_column(1)
self.assertEqual("Ram,Male", dropped.first())
def test_get_duplicates_should_give_duplicates_of_given_column_indexes(self):
rdd = self.sc.parallelize(["Ram,23", "Ram,23", "Jill,45", "Soa,"])
transformable_rdd = TransformableRDD(rdd, 'csv')
duplicates = transformable_rdd.get_duplicates([0])
self.assertEqual("Ram,23", duplicates.first())
def test_transformableRDD_gives_a_count_of_element(self):
rdd = self.sc.parallelize(["2", "3", "4", "5", "6", "7", "7", "7"])
transformable_rdd = TransformableRDD(rdd, 'csv')
self.assertEquals(8, transformable_rdd.count())
def test_transformableRDD_can_impute_the_missing_values_by_ApproxMeanSubstitution(
self):
rdd = self.sc.parallelize(["Ram,9", "Joe,45", "Jill,45", "Soa,"])
transformable_rdd = TransformableRDD(rdd, 'csv')
imputed_rdd = transformable_rdd.impute(1, ApproxMeanSubstitution())
self.assertTrue(imputed_rdd.collect().__contains__("Soa,33.0"))
def test_transformableRDD_can_deduplicate_by_given_column_index(self):
rdd = self.sc.parallelize(["2", "3", "4", "5", "6", "7", "7", "7"])
transformable_rdd = TransformableRDD(rdd, 'csv')
deduplicate_rdd = transformable_rdd.deduplicate([0])
self.assertEquals(6, deduplicate_rdd.count())
def test_drop_column_should_drop_the_given_column(self):
rdd = self.sc.parallelize(["Ram,23,Male", "Ram,23,Male", "Jill,45,Female", "Soa,,Female,"])
transformable_rdd = TransformableRDD(rdd, 'csv')
dropped = transformable_rdd.drop_column(1)
self.assertEqual("Ram,Male", dropped.first())
def test_transformableRDD_gives_a_count_of_element(self):
rdd = self.sc.parallelize(["2", "3", "4", "5", "6", "7", "7", "7"])
transformable_rdd = TransformableRDD(rdd, 'csv')
self.assertEquals(8, transformable_rdd.count())
def test_transformableRDD_can_impute_the_missing_values_by_ApproxMeanSubstitution(self):
rdd = self.sc.parallelize(["Ram,9", "Joe,45", "Jill,45", "Soa,"])
transformable_rdd = TransformableRDD(rdd, 'csv')
imputed_rdd = transformable_rdd.impute(1, ApproxMeanSubstitution())
self.assertTrue(imputed_rdd.collect().__contains__("Soa,33.0"))
def test_list_facets_should_give_facets_of_given_column_indexes(self):
rdd = self.sc.parallelize(["Ram,23,Male", "Ram,23,Male", "Jill,45,Female", "Soa,,Female,"])
transformable_rdd = TransformableRDD(rdd, 'csv')
duplicates = transformable_rdd.list_facets([0, 1, 2])
highest = duplicates.highest()
self.assertEqual("Ram,23,Male", highest[0]._1())
def test_transformableRDD_can_collect_all_the_elements(self):
rdd = self.sc.parallelize(["2", "3", "4", "5", "6", "7", "7", "7"])
transformable_rdd = TransformableRDD(rdd, 'csv')
self.assertEquals(["2", "3", "4", "5", "6", "7", "7", "7"], transformable_rdd.collect())
def test_list_facets_should_give_facets(self):
initial_dataset = self.sc.parallelize(["X,Y", "A,B", "X,Z", "A,Q", "A,E"])
transformable_rdd = TransformableRDD(initial_dataset)
text_facets = transformable_rdd.list_facets_of(0)
self.assertEquals(2, text_facets.count())
def test_transformableRDD_can_collect_all_the_elements(self):
rdd = self.sc.parallelize(["2", "3", "4", "5", "6", "7", "7", "7"])
transformable_rdd = TransformableRDD(rdd, 'csv')
self.assertEquals(["2", "3", "4", "5", "6", "7", "7", "7"],
transformable_rdd.collect())
def test_number_of_column_should_give_number_of_column_of_dataset(self):
initial_dataset = self.sc.parallelize(["1,2", "1,2", "1,3"])
transformable_rdd = TransformableRDD(initial_dataset, "csv")
self.assertEqual(2, transformable_rdd.number_of_columns())
def test_should_give_highest_of_facets(self):
initial_dataset = self.sc.parallelize(["X,Y", "A,B", "X,Z", "A,Q", "A,E"])
transformable_rdd = TransformableRDD(initial_dataset)
text_facets = transformable_rdd.list_facets_of(0)
highest = text_facets.highest()
self.assertEqual("A", highest[0]._1())
def test_transformableRDD_can_deduplicate_by_given_column_index(self):
rdd = self.sc.parallelize(["2", "3", "4", "5", "6", "7", "7", "7"])
transformable_rdd = TransformableRDD(rdd, 'csv')
deduplicate_rdd = transformable_rdd.deduplicate([0])
self.assertEquals(6, deduplicate_rdd.count())
def test_exception_for_text_Facets(self):
initial_dataset = self.sc.parallelize(["1,2", "1,2", "1,3"])
transformable_rdd = TransformableRDD(initial_dataset, "csv")
self.assertRaises(ApplicationException, transformable_rdd.list_facets_of, 4)
def test_number_of_column_should_give_number_of_column_of_dataset(self):
initial_dataset = self.sc.parallelize(["1,2", "1,2", "1,3"])
transformable_rdd = TransformableRDD(initial_dataset, "csv")
self.assertEqual(2, transformable_rdd.number_of_columns())
def test_transformableRDD_can_impute_the_missing_values_by_UnivariateLinearRegressionSubstitution(self):
rdd = self.sc.parallelize(["60,3.1", "61,3.6", "62,3.8", "63,4", "65,4.1", "64,"])
transformable_rdd = TransformableRDD(rdd, 'csv')
imputed_rdd = transformable_rdd.impute(1, UnivariateLinearRegressionSubstitution(0))
self.assertTrue(imputed_rdd.collect().__contains__("64,4.06"))