def scrape_and_save_social_media_from_all_organizations(state_code=''): facebook_pages_found = 0 twitter_handles_found = 0 force_retrieve = False temp_count = 0 organization_manager = OrganizationManager() organization_list_query = Organization.objects.order_by('organization_name') if positive_value_exists(state_code): organization_list_query = organization_list_query.filter(state_served_code=state_code) organization_list = organization_list_query for organization in organization_list: twitter_handle = False facebook_page = False if not organization.organization_website: continue if (not positive_value_exists(organization.organization_twitter_handle)) or force_retrieve: scrape_results = scrape_social_media_from_one_site(organization.organization_website) # Only include a change if we have a new value (do not try to save blank value) if scrape_results['twitter_handle_found'] and positive_value_exists(scrape_results['twitter_handle']): twitter_handle = scrape_results['twitter_handle'] twitter_handles_found += 1 if scrape_results['facebook_page_found'] and positive_value_exists(scrape_results['facebook_page']): facebook_page = scrape_results['facebook_page'] facebook_pages_found += 1 save_results = organization_manager.update_organization_social_media(organization, twitter_handle, facebook_page) if save_results['success']: organization = save_results['organization'] # ###################################### # If we have a Twitter handle for this org, refresh the data if organization.organization_twitter_handle: results = retrieve_twitter_user_info(organization.organization_twitter_handle) if results['success']: save_results = organization_manager.update_organization_twitter_details( organization, results['twitter_json']) if save_results['success']: results = update_social_media_statistics_in_other_tables(organization) # ###################################### # temp_count += 1 # if temp_count > 10: # break status = "ORGANIZATION_SOCIAL_MEDIA_RETRIEVED" results = { 'success': True, 'status': status, 'twitter_handles_found': twitter_handles_found, 'facebook_pages_found': facebook_pages_found, } return results
def scrape_website_for_social_media_view(request, organization_id, force_retrieve=False): authority_required = {'admin'} # admin, verified_volunteer if not voter_has_authority(request, authority_required): return redirect_to_sign_in_page(request, authority_required) facebook_page = False twitter_handle = False organization_manager = OrganizationManager() results = organization_manager.retrieve_organization(organization_id) if not results['organization_found']: messages.add_message(request, messages.INFO, results['status']) return HttpResponseRedirect(reverse('organization:organization_edit', args=(organization_id,))) organization = results['organization'] if not organization.organization_website: messages.add_message(request, messages.ERROR, "No organizational website found.") return HttpResponseRedirect(reverse('organization:organization_position_list', args=(organization_id,))) if (not positive_value_exists(organization.organization_twitter_handle)) or \ (not positive_value_exists(organization.organization_facebook)) or force_retrieve: scrape_results = scrape_social_media_from_one_site(organization.organization_website) if scrape_results['twitter_handle_found']: twitter_handle = scrape_results['twitter_handle'] messages.add_message(request, messages.INFO, "Twitter handle found: " + twitter_handle) else: messages.add_message(request, messages.INFO, "No Twitter handle found: " + scrape_results['status']) if scrape_results['facebook_page_found']: facebook_page = scrape_results['facebook_page'] messages.add_message(request, messages.INFO, "Facebook page found: " + facebook_page) save_results = organization_manager.update_organization_social_media(organization, twitter_handle, facebook_page) if save_results['success']: organization = save_results['organization'] else: organization.organization_twitter_handle = twitter_handle # Store it temporarily # ###################################### if organization.organization_twitter_handle: results = retrieve_twitter_user_info(organization.organization_twitter_handle) if results['success']: save_results = organization_manager.update_organization_twitter_details( organization, results['twitter_json']) if save_results['success']: organization = save_results['organization'] results = update_social_media_statistics_in_other_tables(organization) # ###################################### return HttpResponseRedirect(reverse('organization:organization_position_list', args=(organization_id,)))
def scrape_and_save_social_media_from_all_organizations( state_code='', force_retrieve=False): facebook_pages_found = 0 twitter_handles_found = 0 organization_manager = OrganizationManager() organization_list_query = Organization.objects.order_by( 'organization_name') if positive_value_exists(state_code): organization_list_query = organization_list_query.filter( state_served_code=state_code) organization_list = organization_list_query for organization in organization_list: twitter_handle = False facebook_page = False if not organization.organization_website: continue if (not positive_value_exists( organization.organization_twitter_handle)) or force_retrieve: scrape_results = scrape_social_media_from_one_site( organization.organization_website) # Only include a change if we have a new value (do not try to save blank value) if scrape_results['twitter_handle_found'] and positive_value_exists( scrape_results['twitter_handle']): twitter_handle = scrape_results['twitter_handle'] twitter_handles_found += 1 if scrape_results['facebook_page_found'] and positive_value_exists( scrape_results['facebook_page']): facebook_page = scrape_results['facebook_page'] facebook_pages_found += 1 save_results = organization_manager.update_organization_social_media( organization, twitter_handle, facebook_page) # ###################################### # We refresh the Twitter information in another function status = "ORGANIZATION_SOCIAL_MEDIA_SCRAPED" results = { 'success': True, 'status': status, 'twitter_handles_found': twitter_handles_found, 'facebook_pages_found': facebook_pages_found, } return results
def scrape_and_save_social_media_from_all_organizations(state_code='', force_retrieve=False): facebook_pages_found = 0 twitter_handles_found = 0 organization_manager = OrganizationManager() organization_list_query = Organization.objects.order_by('organization_name') if positive_value_exists(state_code): organization_list_query = organization_list_query.filter(state_served_code=state_code) organization_list = organization_list_query for organization in organization_list: twitter_handle = False facebook_page = False if not organization.organization_website: continue if (not positive_value_exists(organization.organization_twitter_handle)) or force_retrieve: scrape_results = scrape_social_media_from_one_site(organization.organization_website) # Only include a change if we have a new value (do not try to save blank value) if scrape_results['twitter_handle_found'] and positive_value_exists(scrape_results['twitter_handle']): twitter_handle = scrape_results['twitter_handle'] twitter_handles_found += 1 if scrape_results['facebook_page_found'] and positive_value_exists(scrape_results['facebook_page']): facebook_page = scrape_results['facebook_page'] facebook_pages_found += 1 save_results = organization_manager.update_organization_social_media(organization, twitter_handle, facebook_page) # ###################################### # We refresh the Twitter information in another function status = "ORGANIZATION_SOCIAL_MEDIA_SCRAPED" results = { 'success': True, 'status': status, 'twitter_handles_found': twitter_handles_found, 'facebook_pages_found': facebook_pages_found, } return results
def scrape_website_for_social_media_view(request, organization_id, force_retrieve=False): authority_required = {'admin'} # admin, verified_volunteer if not voter_has_authority(request, authority_required): return redirect_to_sign_in_page(request, authority_required) facebook_page = False twitter_handle = False organization_manager = OrganizationManager() results = organization_manager.retrieve_organization(organization_id) if not results['organization_found']: messages.add_message(request, messages.INFO, results['status']) return HttpResponseRedirect( reverse('organization:organization_edit', args=(organization_id, ))) organization = results['organization'] if not organization.organization_website: messages.add_message(request, messages.ERROR, "No organizational website found.") return HttpResponseRedirect( reverse('organization:organization_position_list', args=(organization_id, ))) if (not positive_value_exists(organization.organization_twitter_handle)) or \ (not positive_value_exists(organization.organization_facebook)) or force_retrieve: scrape_results = scrape_social_media_from_one_site( organization.organization_website) if scrape_results['twitter_handle_found']: twitter_handle = scrape_results['twitter_handle'] messages.add_message(request, messages.INFO, "Twitter handle found: " + twitter_handle) else: messages.add_message( request, messages.INFO, "No Twitter handle found: " + scrape_results['status']) if scrape_results['facebook_page_found']: facebook_page = scrape_results['facebook_page'] messages.add_message(request, messages.INFO, "Facebook page found: " + facebook_page) save_results = organization_manager.update_organization_social_media( organization, twitter_handle, facebook_page) if save_results['success']: organization = save_results['organization'] else: organization.organization_twitter_handle = twitter_handle # Store it temporarily # ###################################### if organization.organization_twitter_handle: results = retrieve_twitter_user_info( organization.organization_twitter_handle) if results['success']: save_results = organization_manager.update_organization_twitter_details( organization, results['twitter_json']) if save_results['success']: organization = save_results['organization'] results = update_social_media_statistics_in_other_tables( organization) # ###################################### return HttpResponseRedirect( reverse('organization:organization_position_list', args=(organization_id, )))
def scrape_and_save_social_media_from_all_organizations(state_code=''): facebook_pages_found = 0 twitter_handles_found = 0 force_retrieve = False temp_count = 0 organization_manager = OrganizationManager() organization_list_query = Organization.objects.order_by( 'organization_name') if positive_value_exists(state_code): organization_list_query = organization_list_query.filter( state_served_code=state_code) organization_list = organization_list_query for organization in organization_list: twitter_handle = False facebook_page = False if not organization.organization_website: continue if (not positive_value_exists( organization.organization_twitter_handle)) or force_retrieve: scrape_results = scrape_social_media_from_one_site( organization.organization_website) # Only include a change if we have a new value (do not try to save blank value) if scrape_results['twitter_handle_found'] and positive_value_exists( scrape_results['twitter_handle']): twitter_handle = scrape_results['twitter_handle'] twitter_handles_found += 1 if scrape_results['facebook_page_found'] and positive_value_exists( scrape_results['facebook_page']): facebook_page = scrape_results['facebook_page'] facebook_pages_found += 1 save_results = organization_manager.update_organization_social_media( organization, twitter_handle, facebook_page) if save_results['success']: organization = save_results['organization'] # ###################################### # If we have a Twitter handle for this org, refresh the data if organization.organization_twitter_handle: results = retrieve_twitter_user_info( organization.organization_twitter_handle) if results['success']: save_results = organization_manager.update_organization_twitter_details( organization, results['twitter_json']) if save_results['success']: results = update_social_media_statistics_in_other_tables( organization) # ###################################### # temp_count += 1 # if temp_count > 10: # break status = "ORGANIZATION_SOCIAL_MEDIA_RETRIEVED" results = { 'success': True, 'status': status, 'twitter_handles_found': twitter_handles_found, 'facebook_pages_found': facebook_pages_found, } return results