def test_save_data_to_silo_success(self):
read_file = open('silo/tests/sample_data/test.csv')
reader = CustomDictReader(read_file)
expected_response = {'skipped_rows': set([]), 'num_rows': 4}
result = save_data_to_silo(self.silo, reader, self.read)
self.assertEqual(result, expected_response)
def test_save_data_to_silo_unique_field(self):
read_file = open('silo/tests/sample_data/test.csv')
reader = CustomDictReader(read_file)
lvs = factories.LabelValueStore()
lvs.silo_id = self.silo.id
lvs.save()
expected_response = {
'skipped_rows': set([]),
'num_rows': 1
}
save_data_to_silo(self.silo, reader, self.read)
factories.UniqueFields(name='E-mail', silo=self.silo)
data = [{
'First.Name': 'John',
'Last.Name': 'Lennon',
'E-mail': '*****@*****.**',
}]
result = save_data_to_silo(self.silo, data, self.read)
self.assertEqual(result, expected_response)
lvss = LabelValueStore.objects.filter(silo_id=self.silo.id)
count = 0
for lvs in lvss:
lvs_json = json.loads(lvs.to_json())
if lvs_json.get('First_Name') == 'John':
self.assertEqual(lvs_json.get('Last_Name'), 'Lennon')
count += 1
self.assertEqual(count, 1)
def test_save_data_to_silo_default_read(self):
read_file = open('silo/tests/sample_data/test.csv')
reader = CustomDictReader(read_file)
expected_response = {'skipped_rows': set([]), 'num_rows': 4}
result = save_data_to_silo(self.silo, reader)
lvss = LabelValueStore.objects.filter(silo_id=self.silo.id)
self.assertEqual(result, expected_response)
for lvs in lvss:
self.assertEqual(lvs.read_id, -1)
def test_save_data_to_silo_unique_field_no_lvs(self):
read_file = open('silo/tests/sample_data/test.csv')
reader = CustomDictReader(read_file)
expected_response = {
'skipped_rows': set([]),
'num_rows': 4
}
factories.UniqueFields(name='E-mail', silo=self.silo)
result = save_data_to_silo(self.silo, reader, self.read)
self.assertEqual(result, expected_response)
def test_save_data_to_silo_already_lvs(self):
read_file = open('silo/tests/sample_data/test.csv')
reader = CustomDictReader(read_file)
lvs = factories.LabelValueStore()
lvs.silo_id = self.silo.id
lvs.save()
expected_response = {'skipped_rows': set([]), 'num_rows': 4}
result = save_data_to_silo(self.silo, reader, self.read)
self.assertEqual(result, expected_response)
self.assertEqual(self.silo.data_count, 5)
def process_silo(self, silo_id, read_id):
silo = Silo.objects.get(id=silo_id)
read_obj = Read.objects.get(pk=read_id)
ctype = ContentType.objects.get_for_model(Read)
task = CeleryTask.objects.get(content_type=ctype, object_id=read_obj.id)
task.task_status = CeleryTask.TASK_IN_PROGRESS
task.save()
try:
reader = CustomDictReader(read_obj.file_data)
save_data_to_silo(silo, reader, read_obj)
task.task_status = CeleryTask.TASK_FINISHED
except TypeError, e:
logger.error(e)
task.task_status = CeleryTask.TASK_FAILED