def pull_data():
# Pull data of the active initiatives from the consultation platforms
# and social networks where they are running on
initiatives = Initiative.objects.filter(active=True)
for initiative in initiatives:
invalidate_filters = {'initiative': initiative, 'is_new': False}
try:
invalidate_initiative_content(invalidate_filters,
{'exist_cp': False})
logger.info(
u'Pulling content of the initiative {} from the platform {}'.
format(initiative, initiative.platform))
_pull_content_consultation_platform(initiative.platform,
initiative)
for socialnetwork in initiative.social_network.all():
if not socialnetwork.subscribed_read_time_updates:
try:
invalidate_initiative_content(invalidate_filters,
{'exist_sn': False})
logger.info(
u'Pulling content of the initiative {} from the platform {}'
.format(initiative, socialnetwork))
_pull_content_social_network(socialnetwork, initiative)
except Exception as e:
_handle_pull_exceptions(initiative, socialnetwork,
invalidate_filters,
{'exist_sn': True})
raise AppError(e)
except Exception as e:
_handle_pull_exceptions(initiative, initiative.platform,
invalidate_filters, {'exist_cp': True})
raise AppError(e)
def delete_data():
# Delete comments that don't exist anymore in their original social networks or consultation platforms
logger.info('Checking whether exists comments that do not exist anymore')
unexisting_comments = Comment.objects.filter(
Q(exist_sn=False, sn_id__isnull=False)
| Q(exist_cp=False, cp_id__isnull=False))
for comment in unexisting_comments:
try:
if comment.initiative.active:
if comment.source == 'social_network':
if not comment.source_social.subscribed_read_time_updates:
do_delete_content(comment, 'comment')
else:
do_delete_content(comment, 'comment')
except Exception as e:
if comment.source == 'consultation_platform':
logger.warning(
'Error when trying to delete the comment with the id={} from {}. '
.format(comment.id, comment.source_consultation))
else:
logger.warning(
'Error when trying to delete the comment with the id={} from {}. '
.format(comment.id, comment.source_social))
logger.warning(traceback.format_exc())
# Delete ideas that don't exist anymore in their original social networks or consultation platforms
logger.info('Checking whether exists ideas that do not exist anymore')
unexisting_ideas = Idea.objects.filter(
Q(exist_sn=False, sn_id__isnull=False)
| Q(exist_cp=False, cp_id__isnull=False))
for idea in unexisting_ideas:
try:
if idea.initiative.active:
if idea.source == 'social_network':
if not idea.source_social.subscribed_read_time_updates:
do_delete_content(idea, 'idea')
else:
do_delete_content(idea, 'idea')
except Exception as e:
if idea.source == 'consultation_platform':
logger.warning(
'Error when trying to delete the idea with the id={} from {}. '
.format(idea.id, idea.source_consultation))
else:
logger.warning(
'Error when trying to delete the idea with the id={} from {}. '
.format(idea.id, idea.source_social))
raise AppError(e)
def push_data():
batch_req_ideas = {}
batch_req_comments = {}
# Push ideas to consultation platforms and social networks
logger.info('Pushing ideas to social networks and consultation platforms')
existing_ideas = Idea.objects.filter(Q(exist_sn=True, sn_id__isnull=False) |
Q(exist_cp=True, cp_id__isnull=False)).\
filter(Q(has_changed=True) | Q(is_new=True)).\
order_by('datetime')
tot_ideas_to_sn = Idea.objects.filter(Q(exist_sn=True, sn_id__isnull=False) |
Q(exist_cp=True, cp_id__isnull=False)).\
filter(Q(has_changed=True) | Q(is_new=True)).\
filter(source='consultation_platform').filter(is_new=True).count()
count_ideas_to_sn = 0
for idea in existing_ideas:
try:
if idea.initiative.active:
if idea.source == 'social_network':
if not idea.source_social.subscribed_read_time_updates:
do_push_content(idea, 'idea')
else:
if idea.is_new: count_ideas_to_sn += 1
if count_ideas_to_sn == tot_ideas_to_sn:
batch_req_ideas = do_push_content(
idea, 'idea', True, batch_req_ideas)
else:
batch_req_ideas = do_push_content(
idea, 'idea', False, batch_req_ideas)
except Exception as e:
if idea.source == 'consultation_platform':
logger.warning(
'Error when trying to publish the idea with the id={} on the social networks. '
.format(idea.id))
logger.warning(traceback.format_exc())
else:
logger.warning(
'Error when trying to publish the idea with the id={} on the consultation platforms.'
.format(idea.id))
raise AppError(e)
# Push comments to consultation platforms and social networks
logger.info(
'Pushing comments to social networks and consultation platforms')
existing_comments = Comment.objects.filter(Q(exist_sn=True, sn_id__isnull=False) |
Q(exist_cp=True, cp_id__isnull=False)).\
filter(Q(has_changed=True) | Q(is_new=True)).\
order_by('datetime')
tot_comments_to_sn = Comment.objects.filter(Q(exist_sn=True, sn_id__isnull=False) |
Q(exist_cp=True, cp_id__isnull=False)).\
filter(Q(has_changed=True) | Q(is_new=True)).\
filter(source='consultation_platform').filter(is_new=True).count()
count_comments_to_sn = 0
for comment in existing_comments:
try:
if comment.initiative.active:
if comment.source == 'social_network':
if not comment.source_social.subscribed_read_time_updates:
do_push_content(comment, 'comment')
else:
if comment.is_new: count_comments_to_sn += 1
if count_comments_to_sn == tot_comments_to_sn:
batch_req_comments = do_push_content(
comment, 'comment', True, batch_req_comments)
else:
batch_req_comments = do_push_content(
comment, 'comment', False, batch_req_comments)
except Exception as e:
if comment.source == 'consultation_platform':
logger.warning(
'Error when trying to publish the comment with the id={} on the social networks.'
.format(comment.id))
else:
logger.warning(
'Error when trying to publish the comment with the id={} on the consultation platforms.'
.format(comment.id))
raise AppError(e)