def test_merge_silos_without_mapped_columns(self):
mapping_data = """{
"0": {
"left_table_cols": [],
"right_table_col": "",
"merge_type": ""
},
"left_unmapped_cols": ["first name"],
"right_unmapped_cols": ["last name"]
}"""
left_silo = self._create_silo('left_silo', 1, 'Bob', 'left')
factories.UniqueFields(silo=left_silo, name='first name')
right_silo = self._create_silo('right_silo', 1, 'Marley', 'right')
factories.UniqueFields(silo=right_silo, name='number')
merged_silo = factories.Silo(owner=self.user,
name='merged_silo',
public=True)
response = merge_two_silos(mapping_data, left_silo.pk, right_silo.pk,
merged_silo.pk)
self.assertEqual(
response, {
'status':
"danger",
'message':
"Both silos (%s, %s) must have the same "
"column set as unique fields" %
(left_silo.name, right_silo.name)
})
def test_merge_silo_with_datetime_unique_field(self):
left_silo = self._create_silo('left_silo', 1, 'Bob', 'left')
factories.UniqueFields(silo=left_silo, name='created_date')
right_silo = self._create_silo('right_silo', 1, 'Marley', 'right')
factories.UniqueFields(silo=right_silo, name='created_date')
merged_silo = factories.Silo(owner=self.user,
name='merged_silo',
public=True)
response = merge_two_silos(self.mapping_data, left_silo.pk,
right_silo.pk, merged_silo.pk)
self.assertEqual(response, {
'status': "success",
'message': "Merged data successfully"
})
merged_silo = Silo.objects.get(pk=merged_silo.pk)
self.assertEqual(
LabelValueStore.objects.filter(silo_id=merged_silo.pk).count(), 1)
merged_silo_row = LabelValueStore.objects.get(silo_id=merged_silo.pk)
self.assertEqual(merged_silo_row['first name'], 'Bob')
self.assertEqual(merged_silo_row['last name'], 'Marley')
def test_merge_silo_with_specified_merge_type(self):
mapping_data = """{
"0": {
"left_table_cols": ["number", "points"],
"right_table_col": "number",
"merge_type": "Avg"
},
"left_unmapped_cols": [],
"right_unmapped_cols": []
}"""
user = factories.User(username='******')
left_silo = factories.Silo(owner=user, name='left_silo', public=True)
left_silo_r = factories.LabelValueStore(silo_id=left_silo.pk)
left_silo_r['number'] = 1
left_silo_r['points'] = 5
left_silo_r.save()
left_silo_r2 = factories.LabelValueStore(silo_id=left_silo.pk)
left_silo_r2['number'] = 2
left_silo_r2['points'] = 7
left_silo_r2.save()
factories.UniqueFields(silo=left_silo, name='number')
right_silo = factories.Silo(owner=user, name='right_silo', public=True)
right_silo_r = factories.LabelValueStore(silo_id=right_silo.pk)
right_silo_r['number'] = 1
right_silo_r.save()
right_silo_r2 = factories.LabelValueStore(silo_id=right_silo.pk)
right_silo_r2['number'] = 2
right_silo_r2.save()
factories.UniqueFields(silo=right_silo, name='number')
merged_silo = factories.Silo(owner=user,
name='merged_silo',
public=True)
response = merge_two_silos(mapping_data, left_silo.pk, right_silo.pk,
merged_silo.pk)
self.assertEqual(response, {
'status': "success",
'message': "Merged data successfully"
})
merged_silo = Silo.objects.get(pk=merged_silo.pk)
self.assertEqual(
LabelValueStore.objects.filter(silo_id=merged_silo.pk).count(), 4)
merged_silo_rows = LabelValueStore.objects.filter(
silo_id=merged_silo.pk)
self.assertEqual(merged_silo_rows[0]['number'], 1)
self.assertEqual(merged_silo_rows[1]['number'], 2)
self.assertEqual(merged_silo_rows[2]['number'], 3.0)
self.assertEqual(merged_silo_rows[3]['number'], 4.5)
def test_save_data_to_silo_skipped_rows(self):
read_file = open('silo/tests/sample_data/test.csv')
reader = CustomDictReader(read_file)
lvs = factories.LabelValueStore()
lvs.silo_id = self.silo.id
lvs.save()
save_data_to_silo(self.silo, reader, self.read)
# create multiple lvs
read_file = open('silo/tests/sample_data/test.csv')
reader = CustomDictReader(read_file)
lvs = factories.LabelValueStore()
lvs.silo_id = self.silo.id
lvs.save()
save_data_to_silo(self.silo, reader, self.read)
factories.UniqueFields(name='E-mail', silo=self.silo)
skipped_rows = [
'[email protected]', 'silo_id={}'.format(self.silo.id)
]
expected_response = {'skipped_rows': set(skipped_rows), 'num_rows': 0}
data = [{
'First.Name': 'John',
'Last.Name': 'Lennon',
'E-mail': '*****@*****.**',
}]
result = save_data_to_silo(self.silo, data, self.read)
self.assertEqual(result, expected_response)
def test_save_data_to_silo_unique_field_no_lvs(self):
read_file = open('silo/tests/sample_data/test.csv')
reader = CustomDictReader(read_file)
expected_response = {'skipped_rows': set([]), 'num_rows': 4}
factories.UniqueFields(name='E-mail', silo=self.silo)
result = save_data_to_silo(self.silo, reader, self.read)
self.assertEqual(result, expected_response)
def test_save_data_to_silo_unique_field(self):
read_file = open('silo/tests/sample_data/test.csv')
reader = CustomDictReader(read_file)
lvs = factories.LabelValueStore()
lvs.silo_id = self.silo.id
lvs.save()
expected_response = {'skipped_rows': set([]), 'num_rows': 1}
save_data_to_silo(self.silo, reader, self.read)
factories.UniqueFields(name='E-mail', silo=self.silo)
data = [{
'First.Name': 'John',
'Last.Name': 'Lennon',
'E-mail': '*****@*****.**',
}]
result = save_data_to_silo(self.silo, data, self.read)
self.assertEqual(result, expected_response)
lvss = LabelValueStore.objects.filter(silo_id=self.silo.id)
count = 0
for lvs in lvss:
lvs_json = json.loads(lvs.to_json())
if lvs_json.get('First_Name') == 'John':
self.assertEqual(lvs_json.get('Last_Name'), 'Lennon')
count += 1
self.assertEqual(count, 1)
def test_import_from_gsheet_helper_skipped_rows(
self, mock_get_credential_obj, mock_get_authorized_service,
mock_get_gsheet_metadata, mock_fetch_data_gsheet,
mock_get_or_create_read):
lvs = LabelValueStore()
lvs.silo_id = self.silo.id
lvs.save()
data = [{
'First.Name': 'John',
'Last.Name': 'Doe',
'E-mail': '*****@*****.**',
}, {
'First.Name': 'Bob',
'Last.Name': 'Smith',
'E-mail': '*****@*****.**',
}]
save_data_to_silo(self.silo, data, self.read)
# create multiple lvs
lvs = LabelValueStore()
lvs.silo_id = self.silo.id
lvs.save()
save_data_to_silo(self.silo, data, self.read)
factories.UniqueFields(name='E-mail', silo=self.silo)
data = [
['First.Name', 'Last.Name', 'E-mail'],
['John', 'Lennon', '*****@*****.**'],
]
expected_result = [{
'silo_id': self.silo.id
}, {
'msg':
'Skipped updating/adding records where '
'[email protected],silo_id=1 because there are '
'already multiple records.',
'level':
messages.WARNING
}, {
'msg': 'Operation successful',
'level': messages.SUCCESS
}]
mock_get_credential_obj.return_value = Mock(OAuth2Credentials)
mock_get_authorized_service.return_value = Mock()
mock_get_gsheet_metadata.return_value = (Mock(), None)
mock_fetch_data_gsheet.return_value = (data, None)
mock_get_or_create_read.return_value = self.read
result = gviews_v4.import_from_gsheet_helper(self.tola_user.user,
self.silo.id,
self.silo.name,
1234,
partialcomplete=True)
self.assertEqual(result, ([], expected_result))
def test_import_from_gsheet_helper_with_integer_unique_fields(
self, mock_get_credential_obj, mock_get_authorized_service,
mock_get_gsheet_metadata, mock_fetch_data_gsheet,
mock_get_or_create_read):
'''Import function should update existing data when it got new data with
same unique field'''
lvs = LabelValueStore()
lvs.silo_id = self.silo.id
lvs.save()
data = [{
'First.Name': 'John',
'Last.Name': 'Doe',
'Number': 1,
}, {
'First.Name': 'Bob',
'Last.Name': 'Smith',
'Number': 2,
}]
save_data_to_silo(self.silo, data, self.read)
factories.UniqueFields(name='Number', silo=self.silo)
data = [
['First.Name', 'Last.Name', 'Number'],
['John', 'Lennon', 1],
]
expected_result = [{
'silo_id': self.silo.id
}, {
'msg': 'Operation successful',
'level': messages.SUCCESS
}]
mock_get_credential_obj.return_value = Mock(OAuth2Credentials)
mock_get_authorized_service.return_value = Mock()
mock_get_gsheet_metadata.return_value = (Mock(), None)
mock_fetch_data_gsheet.return_value = (data, None)
mock_get_or_create_read.return_value = self.read
result = gviews_v4.import_from_gsheet_helper(self.hikaya_user.user,
self.silo.id,
self.silo.name, 1234)
self.assertEqual(result, expected_result)
lvss = LabelValueStore.objects.filter(silo_id=self.silo.id)
count = 0
for lvs in lvss:
lvs_json = json.loads(lvs.to_json())
if lvs_json.get('First_Name') == 'John':
self.assertEqual(lvs_json.get('Last_Name'), 'Lennon')
count += 1
self.assertEqual(count, 1)
self.assertEqual(lvss.count(), 3)
def test_merge_silos_without_merged_silo(self):
left_silo = self._create_silo('left_silo', 1, 'Bob', 'left')
factories.UniqueFields(silo=left_silo, name='number')
right_silo = self._create_silo('right_silo', 1, 'Marley', 'right')
factories.UniqueFields(silo=right_silo, name='number')
merged_silo_random_id = random.randint(1, 9999)
response = merge_two_silos(self.mapping_data, left_silo.pk,
right_silo.pk, merged_silo_random_id)
self.assertEqual(
response, {
'status':
"danger",
'message':
"Merged Silo does not exist: "
"silo_id=%s" % merged_silo_random_id
})
def test_import_from_gsheet_helper_unique_fields(
self, mock_get_credential_obj, mock_get_authorized_service,
mock_get_gsheet_metadata, mock_fetch_data_gsheet,
mock_get_or_create_read):
lvs = LabelValueStore()
lvs.silo_id = self.silo.id
lvs.save()
data = [{
'First.Name': 'John',
'Last.Name': 'Doe',
'E-mail': '*****@*****.**',
}, {
'First.Name': 'Bob',
'Last.Name': 'Smith',
'E-mail': '*****@*****.**',
}]
save_data_to_silo(self.silo, data, self.read)
factories.UniqueFields(name='E-mail', silo=self.silo)
data = [
['First.Name', 'Last.Name', 'E-mail'],
['John', 'Lennon', '*****@*****.**'],
]
expected_result = [{
'silo_id': self.silo.id
}, {
'msg': 'Operation successful',
'level': messages.SUCCESS
}]
mock_get_credential_obj.return_value = Mock(OAuth2Credentials)
mock_get_authorized_service.return_value = Mock()
mock_get_gsheet_metadata.return_value = (Mock(), None)
mock_fetch_data_gsheet.return_value = (data, None)
mock_get_or_create_read.return_value = self.read
result = gviews_v4.import_from_gsheet_helper(self.tola_user.user,
self.silo.id,
self.silo.name, 1234)
self.assertEqual(result, expected_result)
lvss = LabelValueStore.objects.filter(silo_id=self.silo.id)
count = 0
for lvs in lvss:
lvs_json = json.loads(lvs.to_json())
if lvs_json.get('First_Name') == 'John':
self.assertEqual(lvs_json.get('Last_Name'), 'Lennon')
count += 1
self.assertEqual(count, 1)
def test_merge_silos_with_different_unique_fields(self):
left_silo = self._create_silo('left_silo', 1, 'Bob', 'left')
factories.UniqueFields(silo=left_silo, name='first name')
right_silo = self._create_silo('right_silo', 1, 'Marley', 'right')
factories.UniqueFields(silo=right_silo, name='number')
merged_silo = factories.Silo(owner=self.user,
name='merged_silo',
public=True)
response = merge_two_silos(self.mapping_data, left_silo.pk,
right_silo.pk, merged_silo.pk)
self.assertEqual(
response, {
'status':
"danger",
'message':
"Both silos (%s, %s) must have the same "
"column set as unique fields" %
(left_silo.name, right_silo.name)
})
def test_merge_silos_without_right_silo(self):
left_silo = self._create_silo('left_silo', 1, 'Bob', 'left')
factories.UniqueFields(silo=left_silo, name='number')
right_silo_random_id = random.randint(1, 9999)
merged_silo = factories.Silo(owner=self.user,
name='merged_silo',
public=True)
response = merge_two_silos(self.mapping_data, left_silo.pk,
right_silo_random_id, merged_silo.pk)
self.assertEqual(
response, {
'status':
"danger",
'message':
"Right Silo does not exist: "
"silo_id=%s" % right_silo_random_id
})
def test_merge_silos_without_unique_field_in_right_silo(self):
left_silo = self._create_silo('left_silo', 1, 'Bob', 'left')
factories.UniqueFields(silo=left_silo, name='number')
right_silo = self._create_silo('right_silo', 1, 'Marley', 'right')
merged_silo = factories.Silo(owner=self.user,
name='merged_silo',
public=True)
response = merge_two_silos(self.mapping_data, left_silo.pk,
right_silo.pk, merged_silo.pk)
self.assertEqual(
response, {
'status':
"danger",
'message':
"The silo, [%s], must have a unique "
"column and it should be the same as the "
"one specified in [%s] silo." %
(right_silo.name, left_silo.name)
})
