def data(self, request, id):
if id <= 0:
return HttpResponseBadRequest("The silo_id = %s is invalid" % id)
silo = Silo.objects.get(pk=id)
if not silo.public:
url = reverse('silos-data', kwargs={'id': silo.pk})
return redirect(u'{}?{}'.format(url, request.GET))
query = request.GET.get('query', "{}")
filter_fields = json.loads(query)
shown_cols = set(
json.loads(
request.GET.get('shown_cols',
json.dumps(getSiloColumnNames(id)))))
# workaround until the problem of javascript
# not increasing the value of length is fixed
data = LabelValueStore.objects(silo_id=id, **filter_fields).exclude(
'create_date', 'edit_date', 'silo_id', 'read_id')
for col in getCompleteSiloColumnNames(id):
if col not in shown_cols:
data = data.exclude(col)
sort = str(request.GET.get('sort', ''))
data = data.order_by(sort)
json_data = json.loads(data.to_json())
return JsonResponse(json_data, safe=False)
def handle(self, *args, **options):
frequency = options['frequency']
if frequency != "daily" and frequency != "weekly":
return self.stdout.write(
"Frequency argument can either be 'daily' or 'weekly'")
# Get all silos that have a unique field setup, have autopush frequency selected and autopush frequency is the same as specified in this command line argument.
silos = Silo.objects.filter(
reads__autopush_frequency__isnull=False,
reads__autopush_frequency=frequency).distinct()
readtypes = ReadType.objects.filter(
Q(read_type__iexact="GSheet Import")
| Q(read_type__iexact='Google Spreadsheet'))
#readtypes = ReadType.objects.filter(reduce(or_, [Q(read_type__iexact="GSheet Import"), Q(read_type__iexact='Google Spreadsheet')] ))
for silo in silos:
try:
reads = silo.reads.filter(
reduce(or_,
[Q(type=read.id) for read in readtypes])).filter(
autopush_frequency__isnull=False,
autopush_frequency=frequency)
cols_to_export = getSiloColumnNames(silo.id)
# query = json.loads(makeQueryForHiddenRow(json.loads(silo.rows_to_hide)))
query = json.loads('{}')
for read in reads:
msgs = export_to_gsheet_helper(silo.owner,
read.resource_id, silo.pk,
query, cols_to_export)
for msg in msgs:
# if it is not a success message then I want to know
if msg.get("level") != 25:
# replace with logger
logger.error(
"silo_id=%s, read_id=%s, level: %s, msg: %s" %
(silo.pk, read.pk, msg.get("level"),
msg.get("msg")))
send_mail("Tola-Tables Auto-Pull Failed",
"table_id: %s, source_id: %s, %s %s" %
(silo.pk, read.pk, msg.get("level"),
msg.get("msg")),
settings.NOTIFICATION_SENDER,
[silo.owner.email],
fail_silently=False)
else:
self.stdout.write(
"Successfully pushed silo_id=%s, read_id=%s." %
(silo.pk, read.pk))
except Exception as e:
logger.error("Silo_id %s encountered the following error: %s" %
(silo.pk, e))
self.stdout.write("Done executing gsheet export command job")
def test_post_edit_column_order(self):
addColsToSilo(self.silo, ['a', 'b', 'c', 'd', 'e', 'f'])
hideSiloColumns(self.silo, ['b', 'e'])
cols_ordered = ['c', 'f', 'a', 'd']
data = {'columns': cols_ordered}
response = self.client.post('/edit_column_order/{}/'.format(
self.silo.pk),
data=data)
self.assertEqual(response.status_code, 302)
self.assertEqual(getSiloColumnNames(self.silo.pk),
['c', 'f', 'a', 'd'])
data = {'columns': cols_ordered}
response = self.client.post('/edit_column_order/0/', data=data)
self.assertEqual(response.status_code, 302)
def test_celery_success(self):
"""
Test if the celery task process_silo actually imports data
:return:
"""
silo = factories.Silo(owner=self.user, public=False)
read_type = factories.ReadType(read_type="CSV")
upload_file = open('silo/tests/sample_data/test.csv', 'rb')
read = factories.Read(owner=self.user,
type=read_type,
file_data=SimpleUploadedFile(
upload_file.name, upload_file.read()))
factories.CeleryTask(task_status=CeleryTask.TASK_CREATED,
content_object=read)
process_done = process_silo(silo.id, read.id)
silo = Silo.objects.get(pk=silo.id)
self.assertEqual(getSiloColumnNames(silo.id),
['First_Name', 'Last_Name', 'E-mail'])
self.assertTrue(process_done)
def export_to_gsheet(request, id):
spreadsheet_id = request.GET.get("resource_id", None)
query = json.loads(request.GET.get('query', "{}"))
if type(query) == list:
query = json.loads(makeQueryForHiddenRow(query))
cols_to_export = json.loads(
request.GET.get('shown_cols', json.dumps(getSiloColumnNames(id))))
cols_to_export.insert(0, '_id')
msgs = export_to_gsheet_helper(request.user, spreadsheet_id, id, query,
cols_to_export)
google_auth_redirect = "export_to_gsheet/%s/" % id
for msg in msgs:
if "redirect_uri_after_step2" in msg.keys():
request.session['redirect_uri_after_step2'] = google_auth_redirect
return HttpResponseRedirect(msg.get("redirect"))
messages.add_message(request, msg.get("level"), msg.get("msg"))
return HttpResponseRedirect(reverse('list_silos'))
def test_postNewFormulaColumn(self):
data = {'math_operation': 'sum', 'column_name': '', 'columns': []}
response = self.client.post('/new_formula_column/{}/'.format(
self.silo.pk),
data=data)
self.assertEqual(response.status_code, 302)
lvs = LabelValueStore()
lvs.a = "1"
lvs.b = "2"
lvs.c = "3"
lvs.silo_id = self.silo.pk
lvs.save()
lvs = LabelValueStore()
lvs.a = "2"
lvs.b = "2"
lvs.c = "3.3"
lvs.silo_id = self.silo.pk
lvs.save()
lvs = LabelValueStore()
lvs.a = "3"
lvs.b = "2"
lvs.c = "hi"
lvs.silo_id = self.silo.pk
lvs.save()
data = {
'math_operation': 'sum',
'column_name': '',
'columns': ['a', 'b', 'c']
}
response = self.client.post('/new_formula_column/{}/'.format(
self.silo.pk),
data=data)
self.assertEqual(response.status_code, 302)
formula_column = self.silo.formulacolumns.get(column_name='sum')
self.assertEqual(formula_column.operation, 'sum')
self.assertEqual(formula_column.mapping, '["a", "b", "c"]')
self.assertEqual(formula_column.column_name, 'sum')
self.assertEqual(getSiloColumnNames(self.silo.pk), ["sum"])
self.silo = Silo.objects.get(pk=self.silo.pk)
self.assertEqual(getColToTypeDict(self.silo).get('sum'), 'float')
try:
lvs = LabelValueStore.objects.get(a="1",
b="2",
c="3",
sum=6.0,
read_id=-1,
silo_id=self.silo.pk)
lvs.delete()
except LabelValueStore.DoesNotExist:
self.assert_(False)
try:
lvs = LabelValueStore.objects.get(a="2",
b="2",
c="3.3",
sum=7.3,
read_id=-1,
silo_id=self.silo.pk)
lvs.delete()
except LabelValueStore.DoesNotExist:
self.assert_(False)
try:
lvs = LabelValueStore.objects.get(a="3",
b="2",
c="hi",
sum="Error",
read_id=-1,
silo_id=self.silo.pk)
lvs.delete()
except LabelValueStore.DoesNotExist:
self.assert_(False)