def handle(self, *args, **options):
silo = None
read = None
silo_id = options['silo_id']
username = options['username']
user = User.objects.get(username__exact=username)
reads = Read.objects.filter(owner=user)
try:
silo = Silo.objects.get(pk=silo_id)
except Silo.DoesNotExist:
raise CommandError('Silo "%s" does not exist' % silo_id)
for read_id in options['read_ids']:
try:
read = reads.filter(pk=read_id)[0]
except Read.DoesNotExist:
raise CommandError('Read "%s" does not exist for user, %s' %
(read_id, user.username))
# Fetch the data from ONA
ona_token = ThirdPartyTokens.objects.get(user=user.pk, name="ONA")
response = requests.get(
read.read_url,
headers={'Authorization': 'Token %s' % ona_token.token})
data = json.loads(response.content)
save_data_to_silo(silo, data, read, user)
self.stdout.write(
'Successfully fetched the READ_ID, "%s", from database' %
read_id)
def test_save_data_to_silo_skipped_rows(self):
read_file = open('silo/tests/sample_data/test.csv')
reader = CustomDictReader(read_file)
lvs = factories.LabelValueStore()
lvs.silo_id = self.silo.id
lvs.save()
save_data_to_silo(self.silo, reader, self.read)
# create multiple lvs
read_file = open('silo/tests/sample_data/test.csv')
reader = CustomDictReader(read_file)
lvs = factories.LabelValueStore()
lvs.silo_id = self.silo.id
lvs.save()
save_data_to_silo(self.silo, reader, self.read)
factories.UniqueFields(name='E-mail', silo=self.silo)
skipped_rows = [
'[email protected]', 'silo_id={}'.format(self.silo.id)
]
expected_response = {'skipped_rows': set(skipped_rows), 'num_rows': 0}
data = [{
'First.Name': 'John',
'Last.Name': 'Lennon',
'E-mail': '*****@*****.**',
}]
result = save_data_to_silo(self.silo, data, self.read)
self.assertEqual(result, expected_response)
def test_save_data_to_silo_unique_field(self):
read_file = open('silo/tests/sample_data/test.csv')
reader = CustomDictReader(read_file)
lvs = factories.LabelValueStore()
lvs.silo_id = self.silo.id
lvs.save()
expected_response = {'skipped_rows': set([]), 'num_rows': 1}
save_data_to_silo(self.silo, reader, self.read)
factories.UniqueFields(name='E-mail', silo=self.silo)
data = [{
'First.Name': 'John',
'Last.Name': 'Lennon',
'E-mail': '*****@*****.**',
}]
result = save_data_to_silo(self.silo, data, self.read)
self.assertEqual(result, expected_response)
lvss = LabelValueStore.objects.filter(silo_id=self.silo.id)
count = 0
for lvs in lvss:
lvs_json = json.loads(lvs.to_json())
if lvs_json.get('First_Name') == 'John':
self.assertEqual(lvs_json.get('Last_Name'), 'Lennon')
count += 1
self.assertEqual(count, 1)
def test_import_from_gsheet_helper_skipped_rows(
self, mock_get_credential_obj, mock_get_authorized_service,
mock_get_gsheet_metadata, mock_fetch_data_gsheet,
mock_get_or_create_read):
lvs = LabelValueStore()
lvs.silo_id = self.silo.id
lvs.save()
data = [{
'First.Name': 'John',
'Last.Name': 'Doe',
'E-mail': '*****@*****.**',
}, {
'First.Name': 'Bob',
'Last.Name': 'Smith',
'E-mail': '*****@*****.**',
}]
save_data_to_silo(self.silo, data, self.read)
# create multiple lvs
lvs = LabelValueStore()
lvs.silo_id = self.silo.id
lvs.save()
save_data_to_silo(self.silo, data, self.read)
factories.UniqueFields(name='E-mail', silo=self.silo)
data = [
['First.Name', 'Last.Name', 'E-mail'],
['John', 'Lennon', '*****@*****.**'],
]
expected_result = [{
'silo_id': self.silo.id
}, {
'msg':
'Skipped updating/adding records where '
'[email protected],silo_id=1 because there are '
'already multiple records.',
'level':
messages.WARNING
}, {
'msg': 'Operation successful',
'level': messages.SUCCESS
}]
mock_get_credential_obj.return_value = Mock(OAuth2Credentials)
mock_get_authorized_service.return_value = Mock()
mock_get_gsheet_metadata.return_value = (Mock(), None)
mock_fetch_data_gsheet.return_value = (data, None)
mock_get_or_create_read.return_value = self.read
result = gviews_v4.import_from_gsheet_helper(self.tola_user.user,
self.silo.id,
self.silo.name,
1234,
partialcomplete=True)
self.assertEqual(result, ([], expected_result))
def test_import_from_gsheet_helper_with_integer_unique_fields(
self, mock_get_credential_obj, mock_get_authorized_service,
mock_get_gsheet_metadata, mock_fetch_data_gsheet,
mock_get_or_create_read):
'''Import function should update existing data when it got new data with
same unique field'''
lvs = LabelValueStore()
lvs.silo_id = self.silo.id
lvs.save()
data = [{
'First.Name': 'John',
'Last.Name': 'Doe',
'Number': 1,
}, {
'First.Name': 'Bob',
'Last.Name': 'Smith',
'Number': 2,
}]
save_data_to_silo(self.silo, data, self.read)
factories.UniqueFields(name='Number', silo=self.silo)
data = [
['First.Name', 'Last.Name', 'Number'],
['John', 'Lennon', 1],
]
expected_result = [{
'silo_id': self.silo.id
}, {
'msg': 'Operation successful',
'level': messages.SUCCESS
}]
mock_get_credential_obj.return_value = Mock(OAuth2Credentials)
mock_get_authorized_service.return_value = Mock()
mock_get_gsheet_metadata.return_value = (Mock(), None)
mock_fetch_data_gsheet.return_value = (data, None)
mock_get_or_create_read.return_value = self.read
result = gviews_v4.import_from_gsheet_helper(self.tola_user.user,
self.silo.id,
self.silo.name, 1234)
self.assertEqual(result, expected_result)
lvss = LabelValueStore.objects.filter(silo_id=self.silo.id)
count = 0
for lvs in lvss:
lvs_json = json.loads(lvs.to_json())
if lvs_json.get('First_Name') == 'John':
self.assertEqual(lvs_json.get('Last_Name'), 'Lennon')
count += 1
self.assertEqual(count, 1)
self.assertEqual(lvss.count(), 3)
def handle(self, *args, **options):
frequency = options['frequency']
if frequency != "daily" and frequency != "weekly":
return self.stdout.write(
"Frequency argument can either be 'daily' or 'weekly'")
silos = Silo.objects.filter(
unique_fields__isnull=False,
reads__autopull_frequency__isnull=False,
reads__autopull_frequency=frequency).distinct()
read_type = ReadType.objects.get(read_type="ONA")
for silo in silos:
reads = silo.reads.filter(type=read_type.pk)
for read in reads:
try:
ona_token = ThirdPartyTokens.objects.get(
user=silo.owner.pk, name="ONA")
except MultipleObjectsReturned as e:
self.logger.error("token error: silo_id=%s, read_id=%s" %
(silo.pk, read.pk))
self.logger.error(e)
continue
try:
response = requests.get(read.read_url,
headers={
'Authorization':
'Token %s' % ona_token.token
},
timeout=10)
except Timeout:
self.logger.error("timeout error: silo_id=%s, read_id=%s" %
(silo.pk, read.pk))
continue
data = json.loads(response.content)
try:
save_data_to_silo(silo, data, read, silo.owner.pk)
self.stdout.write(
'Successfully fetched the READ_ID, "%s", from ONA' %
read.pk)
except TypeError as e:
self.logger.error("type error: silo_id=%s, read_id=%s" %
(silo.pk, read.pk))
self.logger.error(e)
except UnicodeEncodeError as e:
self.logger.error("unicode error: silo_id=%s, read_id=%s" %
(silo.pk, read.pk))
self.logger.error(e)
def test_save_data_to_silo_unique_field_no_lvs(self):
read_file = open('silo/tests/sample_data/test.csv')
reader = CustomDictReader(read_file)
expected_response = {'skipped_rows': set([]), 'num_rows': 4}
factories.UniqueFields(name='E-mail', silo=self.silo)
result = save_data_to_silo(self.silo, reader, self.read)
self.assertEqual(result, expected_response)
def test_save_data_to_silo_success(self):
read_file = open('silo/tests/sample_data/test.csv')
reader = CustomDictReader(read_file)
expected_response = {'skipped_rows': set([]), 'num_rows': 4}
result = save_data_to_silo(self.silo, reader, self.read)
self.assertEqual(result, expected_response)
def process_silo(self, silo_id, read_id):
silo = Silo.objects.get(id=silo_id)
read_obj = Read.objects.get(pk=read_id)
ctype = ContentType.objects.get_for_model(Read)
task = CeleryTask.objects.get(content_type=ctype, object_id=read_obj.id)
task.task_status = CeleryTask.TASK_IN_PROGRESS
task.save()
try:
reader = CustomDictReader(read_obj.file_data)
save_data_to_silo(silo, reader, read_obj)
task.task_status = CeleryTask.TASK_FINISHED
except TypeError, e:
logger.error(e)
task.task_status = CeleryTask.TASK_FAILED
def save_data(self, request):
"""
Persist user input data
"""
if not request.data:
return Response({'detail': 'No data sent.'},
status=status.HTTP_400_BAD_REQUEST)
if 'silo_id' in request.data and 'data' in request.data:
silo_id = request.data['silo_id']
data = request.data['data']
submitted_by_uuid = request.data.get('submitted_by', None)
else:
return Response({'detail': 'Missing data.'},
status=status.HTTP_400_BAD_REQUEST)
if data:
submitted_by = None
# get timestamp
now = datetime.now()
submission_date = now.strftime('%Y-%m-%d')
submission_time = now.strftime('%H:%M:%S')
# if there's a submitted_by uuid, get user's name from db
if submitted_by_uuid:
submitted_by = TolaUser.objects.values_list(
'name', flat=True).get(tola_user_uuid=submitted_by_uuid)
# add values to the default columns
for default_col in self._default_columns:
if default_col['type'] == 'text' and submitted_by:
data.update({'submitted_by': submitted_by})
elif default_col['type'] == 'date':
data.update({'submission_date': submission_date})
elif default_col['type'] == 'time':
data.update({'submission_time': submission_time})
try:
silo = Silo.objects.get(pk=silo_id)
except Silo.DoesNotExist:
return Response({'detail': 'Not found.'},
status=status.HTTP_404_NOT_FOUND)
else:
save_data_to_silo(silo, [data], silo.reads.first())
return Response({'detail': 'It was successfully saved.'},
status=status.HTTP_200_OK)
def test_save_data_to_silo_default_read(self):
read_file = open('silo/tests/sample_data/test.csv')
reader = CustomDictReader(read_file)
expected_response = {'skipped_rows': set([]), 'num_rows': 4}
result = save_data_to_silo(self.silo, reader)
lvss = LabelValueStore.objects.filter(silo_id=self.silo.id)
self.assertEqual(result, expected_response)
for lvs in lvss:
self.assertEqual(lvs.read_id, -1)
def test_save_data_to_silo_list_dict(self):
data = [{
'name': 'John',
'age': '40',
'active': '1957-1980',
'rate': '9.7'
}]
expected_response = {'skipped_rows': set([]), 'num_rows': 1}
result = save_data_to_silo(self.silo, data, self.read)
self.assertEqual(result, expected_response)
def test_save_data_to_silo_already_lvs(self):
read_file = open('silo/tests/sample_data/test.csv')
reader = CustomDictReader(read_file)
lvs = factories.LabelValueStore()
lvs.silo_id = self.silo.id
lvs.save()
expected_response = {'skipped_rows': set([]), 'num_rows': 4}
result = save_data_to_silo(self.silo, reader, self.read)
self.assertEqual(result, expected_response)
self.assertEqual(self.silo.data_count, 5)
def file(request, id):
if request.method == 'POST':
form = UploadForm(request.POST)
if form.is_valid():
read_obj = Read.objects.get(pk=id)
user = User.objects.get(username__exact=request.user)
if request.POST.get("new_silo", None):
silo = Silo(name=request.POST['new_silo'],
owner=user,
public=False,
create_date=timezone.now())
silo.save()
else:
silo = Silo.objects.get(id=request.POST["silo_id"])
# try:
silo.reads.add(read_obj)
silo_id = silo.id
data = json.load(read_obj.file_data)
save_data_to_silo(silo, data, read_obj)
return HttpResponseRedirect('/silo_detail/%s/' % silo_id)
# except Exception as e:
# messages.error(request, "Your JSON file was formatted incorrectly")
# return HttpResponseRedirect('/fileuploadjson')
else:
messages.error(
request,
"There was a problem with reading the contents of your file" +
form.errors)
user = User.objects.get(username__exact=request.user)
# get all of the silo info to pass to the form
get_silo = Silo.objects.filter(owner=user)
# display the form for user to choose a table or ener a new table name to import data into
return render(
request, 'read/file.html', {
'read_id': id,
'form_action': reverse_lazy("fileupload", kwargs={"id": id}),
'get_silo': get_silo,
})
def save_data(self, request):
"""
Persist user input data
"""
if not request.data:
return Response({'detail': 'No data sent.'},
status=status.HTTP_400_BAD_REQUEST)
if 'silo_id' in request.data and 'data' in request.data:
silo_id = request.data['silo_id']
data = request.data['data']
else:
return Response({'detail': 'Missing data.'},
status=status.HTTP_400_BAD_REQUEST)
try:
silo = Silo.objects.get(pk=silo_id)
except Silo.DoesNotExist:
return Response({'detail': 'Not found.'},
status=status.HTTP_404_NOT_FOUND)
else:
save_data_to_silo(silo, [data], silo.reads.first())
return Response({'detail': 'It was successfully saved.'},
status=status.HTTP_200_OK)
def _import_json(self, silo, read):
filename = os.path.join(os.path.dirname(__file__),
'sample_data/moviesbyearnings2013.json')
with open(filename, 'r') as f:
data = json.load(f)
save_data_to_silo(silo, data, read)
def test_save_data_to_silo_empty(self):
expected_response = {'skipped_rows': set([]), 'num_rows': 0}
result = save_data_to_silo(self.silo, list(), self.read)
self.assertEqual(result, expected_response)
