def test_save_data_to_silo_skipped_rows(self):
read_file = open('silo/tests/sample_data/test.csv')
reader = CustomDictReader(read_file)
lvs = factories.LabelValueStore()
lvs.silo_id = self.silo.id
lvs.save()
save_data_to_silo(self.silo, reader, self.read)
# create multiple lvs
read_file = open('silo/tests/sample_data/test.csv')
reader = CustomDictReader(read_file)
lvs = factories.LabelValueStore()
lvs.silo_id = self.silo.id
lvs.save()
save_data_to_silo(self.silo, reader, self.read)
factories.UniqueFields(name='E-mail', silo=self.silo)
skipped_rows = [
'[email protected]', 'silo_id={}'.format(self.silo.id)
]
expected_response = {'skipped_rows': set(skipped_rows), 'num_rows': 0}
data = [{
'First.Name': 'John',
'Last.Name': 'Lennon',
'E-mail': '*****@*****.**',
}]
result = save_data_to_silo(self.silo, data, self.read)
self.assertEqual(result, expected_response)
def test_merge_silo_with_specified_merge_type(self):
mapping_data = """{
"0": {
"left_table_cols": ["number", "points"],
"right_table_col": "number",
"merge_type": "Avg"
},
"left_unmapped_cols": [],
"right_unmapped_cols": []
}"""
user = factories.User(username='******')
left_silo = factories.Silo(owner=user, name='left_silo', public=True)
left_silo_r = factories.LabelValueStore(silo_id=left_silo.pk)
left_silo_r['number'] = 1
left_silo_r['points'] = 5
left_silo_r.save()
left_silo_r2 = factories.LabelValueStore(silo_id=left_silo.pk)
left_silo_r2['number'] = 2
left_silo_r2['points'] = 7
left_silo_r2.save()
factories.UniqueFields(silo=left_silo, name='number')
right_silo = factories.Silo(owner=user, name='right_silo', public=True)
right_silo_r = factories.LabelValueStore(silo_id=right_silo.pk)
right_silo_r['number'] = 1
right_silo_r.save()
right_silo_r2 = factories.LabelValueStore(silo_id=right_silo.pk)
right_silo_r2['number'] = 2
right_silo_r2.save()
factories.UniqueFields(silo=right_silo, name='number')
merged_silo = factories.Silo(owner=user,
name='merged_silo',
public=True)
response = merge_two_silos(mapping_data, left_silo.pk, right_silo.pk,
merged_silo.pk)
self.assertEqual(response, {
'status': "success",
'message': "Merged data successfully"
})
merged_silo = Silo.objects.get(pk=merged_silo.pk)
self.assertEqual(
LabelValueStore.objects.filter(silo_id=merged_silo.pk).count(), 4)
merged_silo_rows = LabelValueStore.objects.filter(
silo_id=merged_silo.pk)
self.assertEqual(merged_silo_rows[0]['number'], 1)
self.assertEqual(merged_silo_rows[1]['number'], 2)
self.assertEqual(merged_silo_rows[2]['number'], 3.0)
self.assertEqual(merged_silo_rows[3]['number'], 4.5)
def test_save_data_to_silo_unique_field(self):
read_file = open('silo/tests/sample_data/test.csv')
reader = CustomDictReader(read_file)
lvs = factories.LabelValueStore()
lvs.silo_id = self.silo.id
lvs.save()
expected_response = {'skipped_rows': set([]), 'num_rows': 1}
save_data_to_silo(self.silo, reader, self.read)
factories.UniqueFields(name='E-mail', silo=self.silo)
data = [{
'First.Name': 'John',
'Last.Name': 'Lennon',
'E-mail': '*****@*****.**',
}]
result = save_data_to_silo(self.silo, data, self.read)
self.assertEqual(result, expected_response)
lvss = LabelValueStore.objects.filter(silo_id=self.silo.id)
count = 0
for lvs in lvss:
lvs_json = json.loads(lvs.to_json())
if lvs_json.get('First_Name') == 'John':
self.assertEqual(lvs_json.get('Last_Name'), 'Lennon')
count += 1
self.assertEqual(count, 1)
def _create_silo(self, name, number, value, option):
silo = factories.Silo(owner=self.user, name=name, public=True)
silo_row = factories.LabelValueStore(silo_id=silo.pk)
silo_row['number'] = number
if option == 'left':
silo_row['first name'] = value
elif option == 'right':
silo_row['last name'] = value
silo_row.save()
return silo
def test_save_data_to_silo_already_lvs(self):
read_file = open('silo/tests/sample_data/test.csv')
reader = CustomDictReader(read_file)
lvs = factories.LabelValueStore()
lvs.silo_id = self.silo.id
lvs.save()
expected_response = {'skipped_rows': set([]), 'num_rows': 4}
result = save_data_to_silo(self.silo, reader, self.read)
self.assertEqual(result, expected_response)
self.assertEqual(self.silo.data_count, 5)