def scrape_and_save_social_media_from_all_organizations(state_code=''):
facebook_pages_found = 0
twitter_handles_found = 0
force_retrieve = False
temp_count = 0
organization_manager = OrganizationManager()
organization_list_query = Organization.objects.order_by('organization_name')
if positive_value_exists(state_code):
organization_list_query = organization_list_query.filter(state_served_code=state_code)
organization_list = organization_list_query
for organization in organization_list:
twitter_handle = False
facebook_page = False
if not organization.organization_website:
continue
if (not positive_value_exists(organization.organization_twitter_handle)) or force_retrieve:
scrape_results = scrape_social_media_from_one_site(organization.organization_website)
# Only include a change if we have a new value (do not try to save blank value)
if scrape_results['twitter_handle_found'] and positive_value_exists(scrape_results['twitter_handle']):
twitter_handle = scrape_results['twitter_handle']
twitter_handles_found += 1
if scrape_results['facebook_page_found'] and positive_value_exists(scrape_results['facebook_page']):
facebook_page = scrape_results['facebook_page']
facebook_pages_found += 1
save_results = organization_manager.update_organization_social_media(organization, twitter_handle,
facebook_page)
if save_results['success']:
organization = save_results['organization']
# ######################################
# If we have a Twitter handle for this org, refresh the data
if organization.organization_twitter_handle:
results = retrieve_twitter_user_info(organization.organization_twitter_handle)
if results['success']:
save_results = organization_manager.update_organization_twitter_details(
organization, results['twitter_json'])
if save_results['success']:
results = update_social_media_statistics_in_other_tables(organization)
# ######################################
# temp_count += 1
# if temp_count > 10:
# break
status = "ORGANIZATION_SOCIAL_MEDIA_RETRIEVED"
results = {
'success': True,
'status': status,
'twitter_handles_found': twitter_handles_found,
'facebook_pages_found': facebook_pages_found,
}
return results
def scrape_website_for_social_media_view(request, organization_id, force_retrieve=False):
authority_required = {'admin'} # admin, verified_volunteer
if not voter_has_authority(request, authority_required):
return redirect_to_sign_in_page(request, authority_required)
facebook_page = False
twitter_handle = False
organization_manager = OrganizationManager()
results = organization_manager.retrieve_organization(organization_id)
if not results['organization_found']:
messages.add_message(request, messages.INFO, results['status'])
return HttpResponseRedirect(reverse('organization:organization_edit', args=(organization_id,)))
organization = results['organization']
if not organization.organization_website:
messages.add_message(request, messages.ERROR, "No organizational website found.")
return HttpResponseRedirect(reverse('organization:organization_position_list', args=(organization_id,)))
if (not positive_value_exists(organization.organization_twitter_handle)) or \
(not positive_value_exists(organization.organization_facebook)) or force_retrieve:
scrape_results = scrape_social_media_from_one_site(organization.organization_website)
if scrape_results['twitter_handle_found']:
twitter_handle = scrape_results['twitter_handle']
messages.add_message(request, messages.INFO, "Twitter handle found: " + twitter_handle)
else:
messages.add_message(request, messages.INFO, "No Twitter handle found: " + scrape_results['status'])
if scrape_results['facebook_page_found']:
facebook_page = scrape_results['facebook_page']
messages.add_message(request, messages.INFO, "Facebook page found: " + facebook_page)
save_results = organization_manager.update_organization_social_media(organization, twitter_handle,
facebook_page)
if save_results['success']:
organization = save_results['organization']
else:
organization.organization_twitter_handle = twitter_handle # Store it temporarily
# ######################################
if organization.organization_twitter_handle:
results = retrieve_twitter_user_info(organization.organization_twitter_handle)
if results['success']:
save_results = organization_manager.update_organization_twitter_details(
organization, results['twitter_json'])
if save_results['success']:
organization = save_results['organization']
results = update_social_media_statistics_in_other_tables(organization)
# ######################################
return HttpResponseRedirect(reverse('organization:organization_position_list', args=(organization_id,)))
def refresh_twitter_organization_details(organization):
organization_manager = OrganizationManager()
if not organization:
status = "ORGANIZATION_TWITTER_DETAILS_NOT_RETRIEVED-ORG_MISSING"
results = {
'success': False,
'status': status,
}
return results
if organization.organization_twitter_handle:
status = "ORGANIZATION_TWITTER_DETAILS-REACHING_OUT_TO_TWITTER"
results = retrieve_twitter_user_info(
organization.organization_twitter_handle)
if results['success']:
status = "ORGANIZATION_TWITTER_DETAILS_RETRIEVED_FROM_TWITTER"
save_results = organization_manager.update_organization_twitter_details(
organization, results['twitter_json'])
if save_results['success']:
results = update_social_media_statistics_in_other_tables(
organization)
status = "ORGANIZATION_TWITTER_DETAILS_RETRIEVED_FROM_TWITTER_AND_SAVED"
else:
status = "ORGANIZATION_TWITTER_DETAILS-CLEARING_DETAILS"
save_results = organization_manager.clear_organization_twitter_details(
organization)
if save_results['success']:
results = update_social_media_statistics_in_other_tables(
organization)
status = "ORGANIZATION_TWITTER_DETAILS_CLEARED_FROM_DB"
results = {
'success': True,
'status': status,
}
return results
def retrieve_twitter_data_for_all_organizations(state_code='',
google_civic_election_id=0,
first_retrieve_only=False):
number_of_twitter_accounts_queried = 0
number_of_organizations_updated = 0
organization_manager = OrganizationManager()
organization_list_query = Organization.objects.order_by(
'organization_name')
if positive_value_exists(state_code):
organization_list_query = organization_list_query.filter(
state_served_code=state_code)
# TODO DALE limit this to organizations that have a voter guide in a particular election
organization_list = organization_list_query
for organization in organization_list:
# ######################################
# If we have a Twitter handle for this org, refresh the data
if organization.organization_twitter_handle:
retrieved_twitter_data = False
if first_retrieve_only:
if not positive_value_exists(
organization.twitter_followers_count):
results = retrieve_twitter_user_info(
organization.organization_twitter_handle)
retrieved_twitter_data = results['success']
number_of_twitter_accounts_queried += 1
else:
results = retrieve_twitter_user_info(
organization.organization_twitter_handle)
retrieved_twitter_data = results['success']
number_of_twitter_accounts_queried += 1
if retrieved_twitter_data:
number_of_organizations_updated += 1
save_results = organization_manager.update_organization_twitter_details(
organization, results['twitter_json'])
if save_results['success']:
results = update_social_media_statistics_in_other_tables(
organization)
status = "ALL_ORGANIZATION_TWITTER_DATA_RETRIEVED"
results = {
'success': True,
'status': status,
'number_of_twitter_accounts_queried':
number_of_twitter_accounts_queried,
'number_of_organizations_updated': number_of_organizations_updated,
}
return results
def refresh_twitter_details(organization):
organization_manager = OrganizationManager()
status = "ENTERING_REFRESH_TWITTER_DETAILS"
if not organization:
status = "ORGANIZATION_TWITTER_DETAILS_NOT_RETRIEVED-ORG_MISSING"
results = {
'success': False,
'status': status,
}
return results
if organization.organization_twitter_handle:
status = "ORGANIZATION_TWITTER_DETAILS-REACHING_OUT_TO_TWITTER"
results = retrieve_twitter_user_info(organization.organization_twitter_handle)
if results['success']:
status = "ORGANIZATION_TWITTER_DETAILS_RETRIEVED_FROM_TWITTER"
save_results = organization_manager.update_organization_twitter_details(
organization, results['twitter_json'])
if save_results['success']:
results = update_social_media_statistics_in_other_tables(organization)
status = "ORGANIZATION_TWITTER_DETAILS_RETRIEVED_FROM_TWITTER_AND_SAVED"
else:
status = "ORGANIZATION_TWITTER_DETAILS-CLEARING_DETAILS"
save_results = organization_manager.clear_organization_twitter_details(organization)
if save_results['success']:
results = update_social_media_statistics_in_other_tables(organization)
status = "ORGANIZATION_TWITTER_DETAILS_CLEARED_FROM_DB"
results = {
'success': True,
'status': status,
}
return results
def retrieve_twitter_data_for_all_organizations(state_code='', google_civic_election_id=0, first_retrieve_only=False):
number_of_twitter_accounts_queried = 0
number_of_organizations_updated = 0
organization_manager = OrganizationManager()
organization_list_query = Organization.objects.order_by('organization_name')
if positive_value_exists(state_code):
organization_list_query = organization_list_query.filter(state_served_code=state_code)
# TODO DALE limit this to organizations that have a voter guide in a particular election
organization_list = organization_list_query
for organization in organization_list:
# ######################################
# If we have a Twitter handle for this org, refresh the data
if organization.organization_twitter_handle:
retrieved_twitter_data = False
if first_retrieve_only:
if not positive_value_exists(organization.twitter_followers_count):
results = retrieve_twitter_user_info(organization.organization_twitter_handle)
retrieved_twitter_data = results['success']
number_of_twitter_accounts_queried += 1
else:
results = retrieve_twitter_user_info(organization.organization_twitter_handle)
retrieved_twitter_data = results['success']
number_of_twitter_accounts_queried += 1
if retrieved_twitter_data:
number_of_organizations_updated += 1
save_results = organization_manager.update_organization_twitter_details(
organization, results['twitter_json'])
if save_results['success']:
results = update_social_media_statistics_in_other_tables(organization)
status = "ALL_ORGANIZATION_TWITTER_DATA_RETRIEVED"
results = {
'success': True,
'status': status,
'number_of_twitter_accounts_queried': number_of_twitter_accounts_queried,
'number_of_organizations_updated': number_of_organizations_updated,
}
return results
def scrape_website_for_social_media_view(request,
organization_id,
force_retrieve=False):
authority_required = {'admin'} # admin, verified_volunteer
if not voter_has_authority(request, authority_required):
return redirect_to_sign_in_page(request, authority_required)
facebook_page = False
twitter_handle = False
organization_manager = OrganizationManager()
results = organization_manager.retrieve_organization(organization_id)
if not results['organization_found']:
messages.add_message(request, messages.INFO, results['status'])
return HttpResponseRedirect(
reverse('organization:organization_edit',
args=(organization_id, )))
organization = results['organization']
if not organization.organization_website:
messages.add_message(request, messages.ERROR,
"No organizational website found.")
return HttpResponseRedirect(
reverse('organization:organization_position_list',
args=(organization_id, )))
if (not positive_value_exists(organization.organization_twitter_handle)) or \
(not positive_value_exists(organization.organization_facebook)) or force_retrieve:
scrape_results = scrape_social_media_from_one_site(
organization.organization_website)
if scrape_results['twitter_handle_found']:
twitter_handle = scrape_results['twitter_handle']
messages.add_message(request, messages.INFO,
"Twitter handle found: " + twitter_handle)
else:
messages.add_message(
request, messages.INFO,
"No Twitter handle found: " + scrape_results['status'])
if scrape_results['facebook_page_found']:
facebook_page = scrape_results['facebook_page']
messages.add_message(request, messages.INFO,
"Facebook page found: " + facebook_page)
save_results = organization_manager.update_organization_social_media(
organization, twitter_handle, facebook_page)
if save_results['success']:
organization = save_results['organization']
else:
organization.organization_twitter_handle = twitter_handle # Store it temporarily
# ######################################
if organization.organization_twitter_handle:
results = retrieve_twitter_user_info(
organization.organization_twitter_handle)
if results['success']:
save_results = organization_manager.update_organization_twitter_details(
organization, results['twitter_json'])
if save_results['success']:
organization = save_results['organization']
results = update_social_media_statistics_in_other_tables(
organization)
# ######################################
return HttpResponseRedirect(
reverse('organization:organization_position_list',
args=(organization_id, )))
def scrape_and_save_social_media_from_all_organizations(state_code=''):
facebook_pages_found = 0
twitter_handles_found = 0
force_retrieve = False
temp_count = 0
organization_manager = OrganizationManager()
organization_list_query = Organization.objects.order_by(
'organization_name')
if positive_value_exists(state_code):
organization_list_query = organization_list_query.filter(
state_served_code=state_code)
organization_list = organization_list_query
for organization in organization_list:
twitter_handle = False
facebook_page = False
if not organization.organization_website:
continue
if (not positive_value_exists(
organization.organization_twitter_handle)) or force_retrieve:
scrape_results = scrape_social_media_from_one_site(
organization.organization_website)
# Only include a change if we have a new value (do not try to save blank value)
if scrape_results['twitter_handle_found'] and positive_value_exists(
scrape_results['twitter_handle']):
twitter_handle = scrape_results['twitter_handle']
twitter_handles_found += 1
if scrape_results['facebook_page_found'] and positive_value_exists(
scrape_results['facebook_page']):
facebook_page = scrape_results['facebook_page']
facebook_pages_found += 1
save_results = organization_manager.update_organization_social_media(
organization, twitter_handle, facebook_page)
if save_results['success']:
organization = save_results['organization']
# ######################################
# If we have a Twitter handle for this org, refresh the data
if organization.organization_twitter_handle:
results = retrieve_twitter_user_info(
organization.organization_twitter_handle)
if results['success']:
save_results = organization_manager.update_organization_twitter_details(
organization, results['twitter_json'])
if save_results['success']:
results = update_social_media_statistics_in_other_tables(
organization)
# ######################################
# temp_count += 1
# if temp_count > 10:
# break
status = "ORGANIZATION_SOCIAL_MEDIA_RETRIEVED"
results = {
'success': True,
'status': status,
'twitter_handles_found': twitter_handles_found,
'facebook_pages_found': facebook_pages_found,
}
return results
