def prepare_dict(self, response_data_dict, request=None):
from apps.fbschema.parse_utils import parse_clean_field_value
'''
Receives the response from fql graph query. Cleans/Parses the data and returns a valid
dict that can be used to create and save a tuple/data-record.
Applied operations on data in this process is in context with the model in concern, hence
it should be called with proper database model, otherwise compare key function will fail
'''
if not compare_keys_with_fields(
self, response_data_dict.keys()
): #TODO: This compare test is happening for each tuple, can we do better ?
raise ValueError #TODO: Use proper exception
model = self # Just to make code more readable
data_dict = {}
fields = [
field for field in model._meta.fields
if getattr(field, 'name') not in model.ignore_fields
]
for field in fields:
field_name = getattr(field, 'name')
key = field_name
value = parse_clean_field_value(field, response_data_dict[key],
request)
data_dict[key] = value
return data_dict
def prepare_dict(self, response_data_dict, request=None):
from apps.fbschema.parse_utils import parse_clean_field_value
'''
Receives the response from fql graph query. Cleans/Parses the data and returns a valid
dict that can be used to create and save a tuple/data-record.
Applied operations on data in this process is in context with the model in concern, hence
it should be called with proper database model, otherwise compare key function will fail
'''
if not compare_keys_with_fields(self, response_data_dict.keys()): #TODO: This compare test is happening for each tuple, can we do better ?
raise ValueError #TODO: Use proper exception
model = self # Just to make code more readable
data_dict = {}
fields = [field for field in model._meta.fields if getattr(field, 'name') not in model.ignore_fields]
for field in fields:
field_name = getattr(field, 'name')
key = field_name
value = parse_clean_field_value(field, response_data_dict[key], request)
data_dict[key] = value
return data_dict
def save_update_delete(self, request, response_data, stream_nature=False):
from apps.fbschema.parse_utils import parse_clean_field_value
'''
Main function which saves, updates and deletes on updated result sets
This function is always called with subclasses
@stream_nature = For example facebook stream is a stream of incoming items. If we compare last 'n' number of database items with incoming
response items and if we don't define stream_nature then existing code will delete those old database items assuming
they don't exist, hence is a stream in which we always save and update, we never delete
@ONLY_SESSION_USER = local_fql_query uses 'owner_identifier' so that it can bring database items in context to compare with response items.
context_uid is usally me() or a freind's uid, but in some tables only session user is allowed so there is no need of
context_uid so we return 'n' last number of records OR all records
'''
if self.owner_identifier == "ONLY_SESSION_USER":
context_uid = None
else:
context_uid = response_data[0][self.owner_identifier]
#Assumptions response_data (list) has some values (dict)
model_data_list = self.local_fql_query(request, context_uid)
primary_identifier = self.primary_identifier
#Response data set of 'primary key', we also need to make sure that data we are compared are 'cleaned'
response_data_set = set([parse_clean_field_value( self.primary_identifier_class(), response_data_dict[primary_identifier] ) \
for response_data_dict in response_data])
#Model data set of 'primary key'
model_data_set = set([
getattr(model_data, primary_identifier)
for model_data in model_data_list
])
#Delete list
if stream_nature:
delete_list = []
else:
delete_list = list(model_data_set.difference(response_data_set))
#Add List
add_list = list(response_data_set.difference(model_data_set))
#Update List
update_list = list(response_data_set.intersection(model_data_set))
print response_data_set
print model_data_set
print add_list
print update_list
for response_data_dict in response_data:
data_dict = self.prepare_dict(response_data_dict, request)
if data_dict[primary_identifier] in add_list:
#Add here
try:
data_tuple = self(user=request.user, **data_dict)
data_tuple.save()
except IntegrityError:
'''
There are some cases where due to facebook data nature we expect integrity error within ids being saved.
This exception should be handled by that particular model which is expecting this case.
A use case : FacebookStream
with some post_id : 'x' changed his profile picture
with same post_id : 'x' and seven others changed profile pictures
'''
logger.info("Integrity Exception handled, Model = %s" %
self.__class__)
print("Integrity Exception handled, Model = %s" %
self.__class__)
data_tuple = self.handle_integrity_exception(
request, data_dict)
data_tuple.save()
elif data_dict[primary_identifier] in update_list:
#Update here
kwargs = {primary_identifier: data_dict[primary_identifier]}
data_tuple = self.objects.get(user=request.user, **kwargs)
data_tuple = self(user=request.user,
id=data_tuple.id,
**data_dict)
data_tuple.save()
def save_update_delete(self, request, response_data, stream_nature=False):
from apps.fbschema.parse_utils import parse_clean_field_value
'''
Main function which saves, updates and deletes on updated result sets
This function is always called with subclasses
@stream_nature = For example facebook stream is a stream of incoming items. If we compare last 'n' number of database items with incoming
response items and if we don't define stream_nature then existing code will delete those old database items assuming
they don't exist, hence is a stream in which we always save and update, we never delete
@ONLY_SESSION_USER = local_fql_query uses 'owner_identifier' so that it can bring database items in context to compare with response items.
context_uid is usally me() or a freind's uid, but in some tables only session user is allowed so there is no need of
context_uid so we return 'n' last number of records OR all records
'''
if self.owner_identifier == "ONLY_SESSION_USER":
context_uid = None
else:
context_uid = response_data[0][self.owner_identifier]
#Assumptions response_data (list) has some values (dict)
model_data_list = self.local_fql_query(request, context_uid)
primary_identifier = self.primary_identifier
#Response data set of 'primary key', we also need to make sure that data we are compared are 'cleaned'
response_data_set = set([parse_clean_field_value( self.primary_identifier_class(), response_data_dict[primary_identifier] ) \
for response_data_dict in response_data])
#Model data set of 'primary key'
model_data_set = set([getattr(model_data, primary_identifier) for model_data in model_data_list])
#Delete list
if stream_nature:
delete_list = []
else:
delete_list = list(model_data_set.difference(response_data_set))
#Add List
add_list = list(response_data_set.difference(model_data_set))
#Update List
update_list = list(response_data_set.intersection(model_data_set))
print response_data_set
print model_data_set
print add_list
print update_list
for response_data_dict in response_data:
data_dict = self.prepare_dict(response_data_dict, request)
if data_dict[primary_identifier] in add_list:
#Add here
try:
data_tuple = self(user=request.user, **data_dict)
data_tuple.save()
except IntegrityError:
'''
There are some cases where due to facebook data nature we expect integrity error within ids being saved.
This exception should be handled by that particular model which is expecting this case.
A use case : FacebookStream
with some post_id : 'x' changed his profile picture
with same post_id : 'x' and seven others changed profile pictures
'''
logger.info("Integrity Exception handled, Model = %s" % self.__class__)
print("Integrity Exception handled, Model = %s" % self.__class__)
data_tuple = self.handle_integrity_exception(request, data_dict)
data_tuple.save()
elif data_dict[primary_identifier] in update_list:
#Update here
kwargs = { primary_identifier : data_dict[primary_identifier] }
data_tuple = self.objects.get(user=request.user, **kwargs)
data_tuple = self(user=request.user, id=data_tuple.id, **data_dict)
data_tuple.save()