def test_import_from_gsheet_helper_skipped_rows(
self, mock_get_credential_obj, mock_get_authorized_service,
mock_get_gsheet_metadata, mock_fetch_data_gsheet,
mock_get_or_create_read):
lvs = LabelValueStore()
lvs.silo_id = self.silo.id
lvs.save()
data = [{
'First.Name': 'John',
'Last.Name': 'Doe',
'E-mail': '*****@*****.**',
}, {
'First.Name': 'Bob',
'Last.Name': 'Smith',
'E-mail': '*****@*****.**',
}]
save_data_to_silo(self.silo, data, self.read)
# create multiple lvs
lvs = LabelValueStore()
lvs.silo_id = self.silo.id
lvs.save()
save_data_to_silo(self.silo, data, self.read)
factories.UniqueFields(name='E-mail', silo=self.silo)
data = [
['First.Name', 'Last.Name', 'E-mail'],
['John', 'Lennon', '*****@*****.**'],
]
expected_result = [{
'silo_id': self.silo.id
}, {
'msg':
'Skipped updating/adding records where '
'[email protected],silo_id=1 because there are '
'already multiple records.',
'level':
messages.WARNING
}, {
'msg': 'Operation successful',
'level': messages.SUCCESS
}]
mock_get_credential_obj.return_value = Mock(OAuth2Credentials)
mock_get_authorized_service.return_value = Mock()
mock_get_gsheet_metadata.return_value = (Mock(), None)
mock_fetch_data_gsheet.return_value = (data, None)
mock_get_or_create_read.return_value = self.read
result = gviews_v4.import_from_gsheet_helper(self.tola_user.user,
self.silo.id,
self.silo.name,
1234,
partialcomplete=True)
self.assertEqual(result, ([], expected_result))
def test_clean_keys(self):
LabelValueStore.objects(silo_id=self.silo.id).delete()
lvs = LabelValueStore()
orig_data = {
'Header 1': 'r1c1',
'create_date': 'r1c3',
'edit_date': 'r1c2',
'_id': 'r1c4'
}
for k, v in orig_data.iteritems():
key = cleanKey(k)
val = smart_str(v, strings_only=True)
key = smart_str(key)
val = val.strip()
setattr(lvs, key, val)
lvs.silo_id = self.silo.id
lvs.save()
returned_data = json.loads(LabelValueStore.objects(
silo_id=self.silo.id).to_json())[0]
returned_data.pop('_id')
expected_data = {
'Header 1': 'r1c1',
'created_date': 'r1c3',
'editted_date': 'r1c2',
'user_assigned_id': 'r1c4',
'read_id': -1,
'silo_id': self.silo.id
}
self.assertEqual(returned_data, expected_data)
LabelValueStore.objects(silo_id=self.silo.id).delete()
def test_import_from_gsheet_helper_with_integer_unique_fields(
self, mock_get_credential_obj, mock_get_authorized_service,
mock_get_gsheet_metadata, mock_fetch_data_gsheet,
mock_get_or_create_read):
'''Import function should update existing data when it got new data with
same unique field'''
lvs = LabelValueStore()
lvs.silo_id = self.silo.id
lvs.save()
data = [{
'First.Name': 'John',
'Last.Name': 'Doe',
'Number': 1,
}, {
'First.Name': 'Bob',
'Last.Name': 'Smith',
'Number': 2,
}]
save_data_to_silo(self.silo, data, self.read)
factories.UniqueFields(name='Number', silo=self.silo)
data = [
['First.Name', 'Last.Name', 'Number'],
['John', 'Lennon', 1],
]
expected_result = [{
'silo_id': self.silo.id
}, {
'msg': 'Operation successful',
'level': messages.SUCCESS
}]
mock_get_credential_obj.return_value = Mock(OAuth2Credentials)
mock_get_authorized_service.return_value = Mock()
mock_get_gsheet_metadata.return_value = (Mock(), None)
mock_fetch_data_gsheet.return_value = (data, None)
mock_get_or_create_read.return_value = self.read
result = gviews_v4.import_from_gsheet_helper(self.hikaya_user.user,
self.silo.id,
self.silo.name, 1234)
self.assertEqual(result, expected_result)
lvss = LabelValueStore.objects.filter(silo_id=self.silo.id)
count = 0
for lvs in lvss:
lvs_json = json.loads(lvs.to_json())
if lvs_json.get('First_Name') == 'John':
self.assertEqual(lvs_json.get('Last_Name'), 'Lennon')
count += 1
self.assertEqual(count, 1)
self.assertEqual(lvss.count(), 3)
def test_import_from_gsheet_helper_unique_fields(
self, mock_get_credential_obj, mock_get_authorized_service,
mock_get_gsheet_metadata, mock_fetch_data_gsheet,
mock_get_or_create_read):
lvs = LabelValueStore()
lvs.silo_id = self.silo.id
lvs.save()
data = [{
'First.Name': 'John',
'Last.Name': 'Doe',
'E-mail': '*****@*****.**',
}, {
'First.Name': 'Bob',
'Last.Name': 'Smith',
'E-mail': '*****@*****.**',
}]
save_data_to_silo(self.silo, data, self.read)
factories.UniqueFields(name='E-mail', silo=self.silo)
data = [
['First.Name', 'Last.Name', 'E-mail'],
['John', 'Lennon', '*****@*****.**'],
]
expected_result = [{
'silo_id': self.silo.id
}, {
'msg': 'Operation successful',
'level': messages.SUCCESS
}]
mock_get_credential_obj.return_value = Mock(OAuth2Credentials)
mock_get_authorized_service.return_value = Mock()
mock_get_gsheet_metadata.return_value = (Mock(), None)
mock_fetch_data_gsheet.return_value = (data, None)
mock_get_or_create_read.return_value = self.read
result = gviews_v4.import_from_gsheet_helper(self.tola_user.user,
self.silo.id,
self.silo.name, 1234)
self.assertEqual(result, expected_result)
lvss = LabelValueStore.objects.filter(silo_id=self.silo.id)
count = 0
for lvs in lvss:
lvs_json = json.loads(lvs.to_json())
if lvs_json.get('First_Name') == 'John':
self.assertEqual(lvs_json.get('Last_Name'), 'Lennon')
count += 1
self.assertEqual(count, 1)
def test_postNewFormulaColumn(self):
data = {
'math_operation': 'sum',
'column_name': '',
'columns': []
}
response = self.client.post('/new_formula_column/{}/'.format(
self.silo.pk), data=data)
self.assertEqual(response.status_code, 302)
lvs = LabelValueStore()
lvs.a = "1"
lvs.b = "2"
lvs.c = "3"
lvs.silo_id = self.silo.pk
lvs.save()
lvs = LabelValueStore()
lvs.a = "2"
lvs.b = "2"
lvs.c = "3.3"
lvs.silo_id = self.silo.pk
lvs.save()
lvs = LabelValueStore()
lvs.a = "3"
lvs.b = "2"
lvs.c = "hi"
lvs.silo_id = self.silo.pk
lvs.save()
data = {
'math_operation': 'sum',
'column_name': '',
'columns': ['a', 'b', 'c']
}
response = self.client.post('/new_formula_column/{}/'.format(
self.silo.pk), data=data)
self.assertEqual(response.status_code, 302)
formula_column = self.silo.formulacolumns.get(column_name='sum')
self.assertEqual(formula_column.operation, 'sum')
self.assertEqual(formula_column.mapping, '["a", "b", "c"]')
self.assertEqual(formula_column.column_name, 'sum')
self.assertEqual(getSiloColumnNames(self.silo.pk), ["sum"])
self.silo = Silo.objects.get(pk=self.silo.pk)
self.assertEqual(getColToTypeDict(self.silo).get('sum'), 'float')
try:
lvs = LabelValueStore.objects.get(a="1", b="2", c="3", sum=6.0,
read_id=-1, silo_id=self.silo.pk)
lvs.delete()
except LabelValueStore.DoesNotExist as e:
self.assert_(False)
try:
lvs = LabelValueStore.objects.get(a="2", b="2", c="3.3", sum=7.3,
read_id=-1, silo_id=self.silo.pk)
lvs.delete()
except LabelValueStore.DoesNotExist as e:
self.assert_(False)
try:
lvs = LabelValueStore.objects.get(a="3", b="2", c="hi", sum="Error",
read_id=-1, silo_id=self.silo.pk)
lvs.delete()
except LabelValueStore.DoesNotExist as e:
self.assert_(False)
def saveDataToSilo(silo, data):
"""
This saves data to the silo
Keyword arguments:
silo -- the silo object, which is meta data for its labe_value_store
data -- a python list of dictionaries. stored in MONGODB
"""
unique_fields = silo.unique_fields.all()
skipped_rows = set()
enc = "latin-1"
for counter, row in enumerate(data):
# reseting filter_criteria for each row
filter_criteria = {}
for uf in unique_fields:
try:
filter_criteria.update({str(uf.name): str(row[uf.name])})
except KeyError:
# when this excpetion occurs, it means that the col identified
# as the unique_col is not present in the fetched dataset
pass
# if filter_criteria is set, then update it with current silo_id
# else set filter_criteria to some non-existent key and value so
# that it triggers a DoesNotExist exception in order to create a new
# document instead of updating an existing one.
if filter_criteria:
filter_criteria.update({'silo_id': silo.id})
else:
filter_criteria.update(
{"nonexistentkey": "NEVER0101010101010NEVER"})
try:
lvs = LabelValueStore.objects.get(**filter_criteria)
#print("updating")
setattr(lvs, "edit_date", timezone.now())
except LabelValueStore.DoesNotExist as e:
lvs = LabelValueStore()
lvs.silo_id = silo.pk
lvs.create_date = timezone.now()
#print("creating")
except LabelValueStore.MultipleObjectsReturned as e:
for k, v in filter_criteria.iteritems():
skipped_rows.add("%s=%s" % (k, v))
#print("skipping")
continue
counter = 0
# set the fields in the curernt document and save it
for key, val in row.iteritems():
if key == "" or key is None or key == "silo_id": continue
elif key == "id" or key == "_id": key = "user_assigned_id"
elif key == "edit_date": key = "editted_date"
elif key == "create_date": key = "created_date"
if type(val) == str or type(val) == unicode:
val = smart_str(val, strings_only=True)
setattr(lvs, key.replace(".", "_").replace("$", "USD"), val)
counter += 1
lvs.save()
combineColumns(silo.pk)
res = {"skipped_rows": skipped_rows, "num_rows": counter}
return res
def save_data_to_silo(silo, data, read=-1, user=None):
"""
This saves data to the silo
Keyword arguments:
silo -- the silo object, which is meta data for its labe_value_store
data -- a python list of dictionaries. stored in MONGODB
read -- the read object, optional only for backwards compatability
user -- an optional parameter to use if its necessary to retrieve
from ThirdPartyTokens
"""
try:
if read.type.read_type == "ONA" and user:
saveOnaDataToSilo(silo, data, read, user)
read_source_id = read.id
except AttributeError:
read_source_id = read
unique_fields = silo.unique_fields.all()
skipped_rows = set()
counter = 0
keys = []
try:
keys = data.fieldnames
keys = [cleanKey(key) for key in keys]
except AttributeError as e:
logger.warning(e)
for counter, row in enumerate(data):
# resetting filter_criteria for each row
filter_criteria = {}
for uf in unique_fields:
try:
filter_criteria.update({str(uf.name): str(row[uf.name])})
except KeyError as e:
# when this excpetion occurs, it means that the col identified
# as the unique_col is not present in the fetched dataset
logger.info(e)
# if filter_criteria is set, then update it with current silo_id
# else set filter_criteria to some non-existent key and value so
# that it triggers a DoesNotExist exception in order to create a new
# document instead of updating an existing one.
if filter_criteria:
filter_criteria.update({'silo_id': silo.id})
try:
lvs = LabelValueStore.objects.get(**filter_criteria)
setattr(lvs, "edit_date", timezone.now())
lvs.read_id = read_source_id
except LabelValueStore.DoesNotExist:
lvs = LabelValueStore()
lvs.silo_id = silo.pk
lvs.create_date = timezone.now()
lvs.read_id = read_source_id
except LabelValueStore.MultipleObjectsReturned:
for k, v in filter_criteria.iteritems():
skipped_rows.add("{}={}".format(str(k), str(v)))
continue
else:
lvs = LabelValueStore()
lvs.silo_id = silo.pk
lvs.create_date = timezone.now()
lvs.read_id = read_source_id
row = clean_data_obj(row)
for key, val in row.iteritems():
if not isinstance(key, tuple):
if key not in keys:
keys.append(key)
setattr(lvs, key, val)
counter += 1
lvs = calculateFormulaCell(lvs, silo)
lvs.save()
addColsToSilo(silo, keys)
res = {"skipped_rows": skipped_rows, "num_rows": counter}
return res
def saveDataToSilo(silo, data, read=-1, user=None):
"""
This saves data to the silo
Keyword arguments:
silo -- the silo object, which is meta data for its labe_value_store
data -- a python list of dictionaries. stored in MONGODB
read -- the read object, optional only for backwards compatability
user -- an optional parameter to use if its necessary to retrieve from ThirdPartyTokens
"""
try:
if read.type.read_type == "ONA" and user:
saveOnaDataToSilo(silo, data, read, user)
read_source_id = read.id
except AttributeError as e:
read_source_id = read
unique_fields = silo.unique_fields.all()
skipped_rows = set()
keys = []
try:
keys = data.fieldnames
keys = [cleanKey(key) for key in keys]
except AttributeError:
pass
fieldToType = getColToTypeDict(silo)
for counter, row in enumerate(data):
# reseting filter_criteria for each row
filter_criteria = {}
for uf in unique_fields:
try:
filter_criteria.update({str(uf.name): str(row[uf.name])})
except KeyError as e:
# when this excpetion occurs, it means that the col identified
# as the unique_col is not present in the fetched dataset
logger.info(e)
# if filter_criteria is set, then update it with current silo_id
# else set filter_criteria to some non-existent key and value so
# that it triggers a DoesNotExist exception in order to create a new
# document instead of updating an existing one.
if filter_criteria:
filter_criteria.update({'silo_id': silo.id})
# else:
# filter_criteria.update({"nonexistentkey":"NEVER0101010101010NEVER"})
try:
lvs = LabelValueStore.objects.get(**filter_criteria)
#print("updating")
setattr(lvs, "edit_date", timezone.now())
lvs.read_id = read_source_id
except LabelValueStore.DoesNotExist as e:
lvs = LabelValueStore()
lvs.silo_id = silo.pk
lvs.create_date = timezone.now()
lvs.read_id = read_source_id
except LabelValueStore.MultipleObjectsReturned as e:
for k, v in filter_criteria.iteritems():
skipped_rows.add("%s=%s" % (str(k), str(v)))
#print("skipping")
continue
else:
lvs = LabelValueStore()
lvs.silo_id = silo.pk
lvs.create_date = timezone.now()
lvs.read_id = read_source_id
counter = 0
row = cleanDataObj(row, silo)
for key, val in row.iteritems():
# if key == "" or key is None or key == "silo_id": continue
# elif key == "id" or key == "_id": key = "user_assigned_id"
# elif key == "edit_date": key = "editted_date"
# elif key == "create_date": key = "created_date"
# if type(val) == str or type(val) == unicode:
# val = smart_str(val, strings_only=True).strip()
# if fieldToType.get(key, 'string') == 'int':
# try:
# val = int(val)
# except ValueError as e:
# continue
# if fieldToType.get(key, 'string') == 'double':
# try:
# val = float(val)
# except ValueError as e:
# continue
if not isinstance(key, tuple):
if key not in keys:
keys.append(key)
setattr(lvs, key, val)
counter += 1
lvs = calculateFormulaCell(lvs, silo)
lvs.save()
addColsToSilo(silo, keys)
res = {"skipped_rows": skipped_rows, "num_rows": counter}
return res
