def scrape_and_save_social_media_from_all_organizations(state_code=''): facebook_pages_found = 0 twitter_handles_found = 0 force_retrieve = False temp_count = 0 organization_manager = OrganizationManager() organization_list_query = Organization.objects.order_by('organization_name') if positive_value_exists(state_code): organization_list_query = organization_list_query.filter(state_served_code=state_code) organization_list = organization_list_query for organization in organization_list: twitter_handle = False facebook_page = False if not organization.organization_website: continue if (not positive_value_exists(organization.organization_twitter_handle)) or force_retrieve: scrape_results = scrape_social_media_from_one_site(organization.organization_website) # Only include a change if we have a new value (do not try to save blank value) if scrape_results['twitter_handle_found'] and positive_value_exists(scrape_results['twitter_handle']): twitter_handle = scrape_results['twitter_handle'] twitter_handles_found += 1 if scrape_results['facebook_page_found'] and positive_value_exists(scrape_results['facebook_page']): facebook_page = scrape_results['facebook_page'] facebook_pages_found += 1 save_results = organization_manager.update_organization_social_media(organization, twitter_handle, facebook_page) if save_results['success']: organization = save_results['organization'] # ###################################### # If we have a Twitter handle for this org, refresh the data if organization.organization_twitter_handle: results = retrieve_twitter_user_info(organization.organization_twitter_handle) if results['success']: save_results = organization_manager.update_organization_twitter_details( organization, results['twitter_json']) if save_results['success']: results = update_social_media_statistics_in_other_tables(organization) # ###################################### # temp_count += 1 # if temp_count > 10: # break status = "ORGANIZATION_SOCIAL_MEDIA_RETRIEVED" results = { 'success': True, 'status': status, 'twitter_handles_found': twitter_handles_found, 'facebook_pages_found': facebook_pages_found, } return results
def scrape_website_for_social_media_view(request, organization_id, force_retrieve=False): authority_required = {'admin'} # admin, verified_volunteer if not voter_has_authority(request, authority_required): return redirect_to_sign_in_page(request, authority_required) facebook_page = False twitter_handle = False organization_manager = OrganizationManager() results = organization_manager.retrieve_organization(organization_id) if not results['organization_found']: messages.add_message(request, messages.INFO, results['status']) return HttpResponseRedirect(reverse('organization:organization_edit', args=(organization_id,))) organization = results['organization'] if not organization.organization_website: messages.add_message(request, messages.ERROR, "No organizational website found.") return HttpResponseRedirect(reverse('organization:organization_position_list', args=(organization_id,))) if (not positive_value_exists(organization.organization_twitter_handle)) or \ (not positive_value_exists(organization.organization_facebook)) or force_retrieve: scrape_results = scrape_social_media_from_one_site(organization.organization_website) if scrape_results['twitter_handle_found']: twitter_handle = scrape_results['twitter_handle'] messages.add_message(request, messages.INFO, "Twitter handle found: " + twitter_handle) else: messages.add_message(request, messages.INFO, "No Twitter handle found: " + scrape_results['status']) if scrape_results['facebook_page_found']: facebook_page = scrape_results['facebook_page'] messages.add_message(request, messages.INFO, "Facebook page found: " + facebook_page) save_results = organization_manager.update_organization_social_media(organization, twitter_handle, facebook_page) if save_results['success']: organization = save_results['organization'] else: organization.organization_twitter_handle = twitter_handle # Store it temporarily # ###################################### if organization.organization_twitter_handle: results = retrieve_twitter_user_info(organization.organization_twitter_handle) if results['success']: save_results = organization_manager.update_organization_twitter_details( organization, results['twitter_json']) if save_results['success']: organization = save_results['organization'] results = update_social_media_statistics_in_other_tables(organization) # ###################################### return HttpResponseRedirect(reverse('organization:organization_position_list', args=(organization_id,)))
def retrieve_twitter_data_for_all_organizations(state_code='', google_civic_election_id=0, first_retrieve_only=False): number_of_twitter_accounts_queried = 0 number_of_organizations_updated = 0 organization_manager = OrganizationManager() organization_list_query = Organization.objects.order_by( 'organization_name') if positive_value_exists(state_code): organization_list_query = organization_list_query.filter( state_served_code=state_code) # TODO DALE limit this to organizations that have a voter guide in a particular election organization_list = organization_list_query for organization in organization_list: # ###################################### # If we have a Twitter handle for this org, refresh the data if organization.organization_twitter_handle: retrieved_twitter_data = False if first_retrieve_only: if not positive_value_exists( organization.twitter_followers_count): results = retrieve_twitter_user_info( organization.organization_twitter_handle) retrieved_twitter_data = results['success'] number_of_twitter_accounts_queried += 1 else: results = retrieve_twitter_user_info( organization.organization_twitter_handle) retrieved_twitter_data = results['success'] number_of_twitter_accounts_queried += 1 if retrieved_twitter_data: number_of_organizations_updated += 1 save_results = organization_manager.update_organization_twitter_details( organization, results['twitter_json']) if save_results['success']: results = update_social_media_statistics_in_other_tables( organization) status = "ALL_ORGANIZATION_TWITTER_DATA_RETRIEVED" results = { 'success': True, 'status': status, 'number_of_twitter_accounts_queried': number_of_twitter_accounts_queried, 'number_of_organizations_updated': number_of_organizations_updated, } return results
def retrieve_twitter_data_for_all_organizations(state_code='', google_civic_election_id=0, first_retrieve_only=False): number_of_twitter_accounts_queried = 0 number_of_organizations_updated = 0 organization_manager = OrganizationManager() organization_list_query = Organization.objects.order_by('organization_name') if positive_value_exists(state_code): organization_list_query = organization_list_query.filter(state_served_code=state_code) # TODO DALE limit this to organizations that have a voter guide in a particular election organization_list = organization_list_query for organization in organization_list: # ###################################### # If we have a Twitter handle for this org, refresh the data if organization.organization_twitter_handle: retrieved_twitter_data = False if first_retrieve_only: if not positive_value_exists(organization.twitter_followers_count): results = retrieve_twitter_user_info(organization.organization_twitter_handle) retrieved_twitter_data = results['success'] number_of_twitter_accounts_queried += 1 else: results = retrieve_twitter_user_info(organization.organization_twitter_handle) retrieved_twitter_data = results['success'] number_of_twitter_accounts_queried += 1 if retrieved_twitter_data: number_of_organizations_updated += 1 save_results = organization_manager.update_organization_twitter_details( organization, results['twitter_json']) if save_results['success']: results = update_social_media_statistics_in_other_tables(organization) status = "ALL_ORGANIZATION_TWITTER_DATA_RETRIEVED" results = { 'success': True, 'status': status, 'number_of_twitter_accounts_queried': number_of_twitter_accounts_queried, 'number_of_organizations_updated': number_of_organizations_updated, } return results
def refresh_twitter_organization_details(organization): organization_manager = OrganizationManager() if not organization: status = "ORGANIZATION_TWITTER_DETAILS_NOT_RETRIEVED-ORG_MISSING" results = { 'success': False, 'status': status, } return results if organization.organization_twitter_handle: status = "ORGANIZATION_TWITTER_DETAILS-REACHING_OUT_TO_TWITTER" results = retrieve_twitter_user_info( organization.organization_twitter_handle) if results['success']: status = "ORGANIZATION_TWITTER_DETAILS_RETRIEVED_FROM_TWITTER" save_results = organization_manager.update_organization_twitter_details( organization, results['twitter_json']) if save_results['success']: results = update_social_media_statistics_in_other_tables( organization) status = "ORGANIZATION_TWITTER_DETAILS_RETRIEVED_FROM_TWITTER_AND_SAVED" else: status = "ORGANIZATION_TWITTER_DETAILS-CLEARING_DETAILS" save_results = organization_manager.clear_organization_twitter_details( organization) if save_results['success']: results = update_social_media_statistics_in_other_tables( organization) status = "ORGANIZATION_TWITTER_DETAILS_CLEARED_FROM_DB" results = { 'success': True, 'status': status, } return results
def refresh_twitter_details(organization): organization_manager = OrganizationManager() status = "ENTERING_REFRESH_TWITTER_DETAILS" if not organization: status = "ORGANIZATION_TWITTER_DETAILS_NOT_RETRIEVED-ORG_MISSING" results = { 'success': False, 'status': status, } return results if organization.organization_twitter_handle: status = "ORGANIZATION_TWITTER_DETAILS-REACHING_OUT_TO_TWITTER" results = retrieve_twitter_user_info(organization.organization_twitter_handle) if results['success']: status = "ORGANIZATION_TWITTER_DETAILS_RETRIEVED_FROM_TWITTER" save_results = organization_manager.update_organization_twitter_details( organization, results['twitter_json']) if save_results['success']: results = update_social_media_statistics_in_other_tables(organization) status = "ORGANIZATION_TWITTER_DETAILS_RETRIEVED_FROM_TWITTER_AND_SAVED" else: status = "ORGANIZATION_TWITTER_DETAILS-CLEARING_DETAILS" save_results = organization_manager.clear_organization_twitter_details(organization) if save_results['success']: results = update_social_media_statistics_in_other_tables(organization) status = "ORGANIZATION_TWITTER_DETAILS_CLEARED_FROM_DB" results = { 'success': True, 'status': status, } return results
def scrape_website_for_social_media_view(request, organization_id, force_retrieve=False): authority_required = {'admin'} # admin, verified_volunteer if not voter_has_authority(request, authority_required): return redirect_to_sign_in_page(request, authority_required) facebook_page = False twitter_handle = False organization_manager = OrganizationManager() results = organization_manager.retrieve_organization(organization_id) if not results['organization_found']: messages.add_message(request, messages.INFO, results['status']) return HttpResponseRedirect( reverse('organization:organization_edit', args=(organization_id, ))) organization = results['organization'] if not organization.organization_website: messages.add_message(request, messages.ERROR, "No organizational website found.") return HttpResponseRedirect( reverse('organization:organization_position_list', args=(organization_id, ))) if (not positive_value_exists(organization.organization_twitter_handle)) or \ (not positive_value_exists(organization.organization_facebook)) or force_retrieve: scrape_results = scrape_social_media_from_one_site( organization.organization_website) if scrape_results['twitter_handle_found']: twitter_handle = scrape_results['twitter_handle'] messages.add_message(request, messages.INFO, "Twitter handle found: " + twitter_handle) else: messages.add_message( request, messages.INFO, "No Twitter handle found: " + scrape_results['status']) if scrape_results['facebook_page_found']: facebook_page = scrape_results['facebook_page'] messages.add_message(request, messages.INFO, "Facebook page found: " + facebook_page) save_results = organization_manager.update_organization_social_media( organization, twitter_handle, facebook_page) if save_results['success']: organization = save_results['organization'] else: organization.organization_twitter_handle = twitter_handle # Store it temporarily # ###################################### if organization.organization_twitter_handle: results = retrieve_twitter_user_info( organization.organization_twitter_handle) if results['success']: save_results = organization_manager.update_organization_twitter_details( organization, results['twitter_json']) if save_results['success']: organization = save_results['organization'] results = update_social_media_statistics_in_other_tables( organization) # ###################################### return HttpResponseRedirect( reverse('organization:organization_position_list', args=(organization_id, )))
def scrape_and_save_social_media_from_all_organizations(state_code=''): facebook_pages_found = 0 twitter_handles_found = 0 force_retrieve = False temp_count = 0 organization_manager = OrganizationManager() organization_list_query = Organization.objects.order_by( 'organization_name') if positive_value_exists(state_code): organization_list_query = organization_list_query.filter( state_served_code=state_code) organization_list = organization_list_query for organization in organization_list: twitter_handle = False facebook_page = False if not organization.organization_website: continue if (not positive_value_exists( organization.organization_twitter_handle)) or force_retrieve: scrape_results = scrape_social_media_from_one_site( organization.organization_website) # Only include a change if we have a new value (do not try to save blank value) if scrape_results['twitter_handle_found'] and positive_value_exists( scrape_results['twitter_handle']): twitter_handle = scrape_results['twitter_handle'] twitter_handles_found += 1 if scrape_results['facebook_page_found'] and positive_value_exists( scrape_results['facebook_page']): facebook_page = scrape_results['facebook_page'] facebook_pages_found += 1 save_results = organization_manager.update_organization_social_media( organization, twitter_handle, facebook_page) if save_results['success']: organization = save_results['organization'] # ###################################### # If we have a Twitter handle for this org, refresh the data if organization.organization_twitter_handle: results = retrieve_twitter_user_info( organization.organization_twitter_handle) if results['success']: save_results = organization_manager.update_organization_twitter_details( organization, results['twitter_json']) if save_results['success']: results = update_social_media_statistics_in_other_tables( organization) # ###################################### # temp_count += 1 # if temp_count > 10: # break status = "ORGANIZATION_SOCIAL_MEDIA_RETRIEVED" results = { 'success': True, 'status': status, 'twitter_handles_found': twitter_handles_found, 'facebook_pages_found': facebook_pages_found, } return results