def combine_children(websites, texts, classes, child_links, child_text):
    combined_sites = {'websites': [], 'texts': [], 'classes': []}
    sites_classes = zip(websites, classes)

    for site, cls in sites_classes:
        group = filter(lambda s: class_order(
            conversions.page_relation(s[0], False, site),
            'False') > 0, zip(websites, classes, texts)
        )

        try:
            min_tuple = min(group, key=lambda g: class_ranks[g[1]])
        except:
            import pdb
            pdb.set_trace()

        group_class = min_tuple[1] + 'Combined'
        group_site = min_tuple[0]
        group_children_text = []
        child_site_texts = zip(child_links, child_text)

        for child, ctext in child_site_texts:
            if conversions.child_or_equal_page(group_site, child, True):
                group_children_text.append(ctext)
                child_idx = child_links.index(child)
                child_links.pop(child_idx)
                child_text.pop(child_idx)
                child_site_texts.pop(child_idx)

        combined_sites['websites'].append(group_site)
        combined_sites['classes'].append(group_class)

        group_text = ' '.join(group_children_text)

        for g in group:
            site_idx = websites.index(g[0])
            websites.pop(site_idx)
            classes.pop(site_idx)
            sites_classes.pop(site_idx)
            texts.pop(site_idx)
        group_text += ' ' + ' '.join(map(lambda g: g[2], group))
        combined_sites['texts'].append(group_text)

    return (combined_sites['websites'],
            combined_sites['texts'],
            combined_sites['classes'])
def getlinks(candidate, webpage, state, district_type, district_name):
    district_type = district_type.replace('_',' ').strip()
    state = state_map[state.strip()]
    candidate, last, first = conversions.clean_name(candidate)
    candidate = '+'.join(candidate.split(' '))
    print candidate
    state = '+'.join(state.split(' '))
    district_type = '+'.join(district_type.split(' '))
    district_name = '+'.join(district_name.strip().split(' '))
    search_urls = []
    extra_children_searches = []
    precise_searches = []
    search_urls.append(u'https://www.googleapis.com/customsearch/v1?cx=011743744063680272768:cp4-iesopjm&key=AIzaSyCdHlGJuMzGBH9hNsEMObffDIkzJ44EQhA&hl=en&q={name}+{state}'.format(name=candidate, state=state))
    extra_children_searches.append(u'https://www.googleapis.com/customsearch/v1?cx=011743744063680272768:cp4-iesopjm&key=AIzaSyCdHlGJuMzGBH9hNsEMObffDIkzJ44EQhA&hl=en&q={name}+{state}+info'.format(name=candidate, state=state))
    extra_children_searches.append(u'https://www.googleapis.com/customsearch/v1?cx=011743744063680272768:cp4-iesopjm&key=AIzaSyCdHlGJuMzGBH9hNsEMObffDIkzJ44EQhA&hl=en&q={name}+{state}+sk=info'.format(name=candidate, state=state))
    precise_searches.append(u'https://www.googleapis.com/customsearch/v1?cx=011743744063680272768:cp4-iesopjm&key=AIzaSyCdHlGJuMzGBH9hNsEMObffDIkzJ44EQhA&hl=en&q={name}+{state}+campaign'.format(name=candidate, state=state))
    precise_searches.append(u'https://www.googleapis.com/customsearch/v1?cx=011743744063680272768:cp4-iesopjm&key=AIzaSyCdHlGJuMzGBH9hNsEMObffDIkzJ44EQhA&hl=en&q={name}+{state}+elect'.format(name=candidate, state=state))
    search_urls = [s.encode(chardet.detect(s.encode('utf-8'))['encoding']) for s in search_urls]
    extra_children_searches = [s.encode(chardet.detect(s.encode('utf-8'))['encoding']) for s in extra_children_searches]
    precise_searches = [s.encode(chardet.detect(s.encode('utf-8'))['encoding']) for s in precise_searches]
    old_webpage = webpage
    if webpage != 'www.gernensamples.com':
        webpage = conversions.get_redirect(webpage)
    #if webpage == '404' or webpage == 'ERROR':
        #raise Exception
    websites = []
    webpage_stripped = re.match(r'(?:https?://)?(?:www\.)?(?P<content>.+)',webpage).groupdict()['content'].rstrip('/')
    old_webpage_stripped = re.match(r'(?:https?://)?(?:www\.)?(?P<content>.+)',old_webpage).groupdict()['content'].rstrip('/')
    #TODO strip queries
    webpage_no_queries = ul.urlparse.urlparse(webpage)
    webpage_no_queries = re.match(r'(?:www\.)?(?P<content>.+)',webpage_no_queries.netloc + webpage_no_queries.path).groupdict()['content'].rstrip('/')
    old_webpage_no_queries = ul.urlparse.urlparse(old_webpage)
    old_webpage_no_queries = re.match(r'(?:www\.)?(?P<content>.+)',old_webpage_no_queries.netloc + old_webpage_no_queries.path).groupdict()['content'].rstrip('/')
    patt = re.compile(r'^https?://(?:www.)?{webpage}/?$'.format(webpage=webpage_stripped.lower()))
    old_patt = re.compile(r'^https?://(?:www.)?{webpage}/?$'.format(webpage=old_webpage_stripped.lower()))
    child_patt = re.compile(r'^https?://(?:www\.)?{webpage}.+'.format(webpage=webpage_no_queries.lower()))
    old_child_patt = re.compile(r'^https?://(?:www\.)?{webpage}.+'.format(webpage=old_webpage_no_queries.lower()))
    n = 4
    while True:
        results = map(lambda x: json.loads(requests.get(x).text),search_urls)
        if any(map(lambda r: r.has_key('error') and (r['error']['code'] == 403 or r['error']['code'] == 503),results)):
            print 'sleeping'
            time.sleep(n + random.randint(1,1000)/1000.)
            n = n*2
        elif any(map(lambda r: r.has_key('error'), results)):
            raise Exception(', '.join(map(lambda r: r['error']['message'], filter(lambda r: r.has_key('error'),results))))
        else:
            break
    n = 4
    while True:
        child_results = map(lambda x: json.loads(requests.get(x).text),extra_children_searches)
        if any(map(lambda r: r.has_key('error') and (r['error']['code'] == 403 or r['error']['code'] == 503),child_results)):
            print 'sleeping'
            time.sleep(n + random.randint(1,1000)/1000.)
            n = n*2
        elif any(map(lambda r: r.has_key('error'), child_results)):
            raise Exception(', '.join(map(lambda r: r['error']['message'], filter(lambda r: r.has_key('error'),child_results))))
        else:
            break
    n = 4
    while True:
        precise_results = map(lambda x: json.loads(requests.get(x).text),precise_searches)
        if any(map(lambda r: r.has_key('error') and (r['error']['code'] == 403 or r['error']['code'] == 503),precise_results)):
            print 'sleeping'
            time.sleep(n + random.randint(1,1000)/1000.)
            n = n*2
        elif any(map(lambda r: r.has_key('error'), precise_results)):
            raise Exception(', '.join(map(lambda r: r['error']['message'], filter(lambda r: r.has_key('error'),precise_results))))
        else:
            break

    if type(results) != list:
        print type(results)
        results = [results]
    real_results = [(r if r.has_key('items') else {'items':[]}) for r in results]
    results = real_results
    search_links = [[i['link'].lower() for i in r['items']] for r in results]
    search_text = [[u'{title} {link} {pagemap} {snippet}'.format(**convert_pagemap_dict(i)).lower().encode('utf-8') for i in r['items']] for r in results]
    for ri in range(len(search_links)):
        for si in range(len(search_links[ri])):
            for r in precise_results:
                if r.has_key('items'):
                    for i in r['items']:
                        if conversions.child_or_equal_page(search_links[ri][si], i['link'].lower(), True):
                            search_text[ri][si] += ' bipspecialappearsinprecise'
    child_links = [i['link'].lower() for r in child_results if r.has_key('items') for i in r['items']]
    child_text = [u'{title} {link} {pagemap} {snippet}'.format(**convert_pagemap_dict(i)).lower().encode('utf-8') for r in child_results if r.has_key('items') for i in r['items']]
    #search_text = [[u'{title} {link} {pagemap} {snippet}'.format(**i).lower().encode('utf-8') for i in r['items']] for r in results]
    search_class = [map(lambda s: conversions.page_relation(s, True, webpage,old_webpage),sl) for sl in search_links]
    #search_class = [map(lambda s: 'True' if patt.match(s) != None or old_patt.match(s) != None else ('Child' if child_patt.match(s) != None or old_child_patt.match(s) != None else 'False'),sl) for sl in search_links]
    #print search_text
    #TODO Clean up ssv code
    ssv = [any(map(patt.match,sl)) or any(map(old_patt.match,sl)) for sl in search_links]
    non_websites = [[i['link'] for i in r['items'] if webpage not in i['link']] for r in results]
    cs,ct,cc = zip(*[combine_children(search_links[i],search_text[i],search_class[i], child_links, child_text) for i in range(len(search_links))])
    print 'got there',len(results[0]['items'])
    return non_websites, ssv, webpage_stripped, search_links, search_text, [r['items'] for r in results], search_class, cs, ct, cc,child_links,child_text
def combine_children(websites, texts, classes, child_links, child_text):
    combined_sites = {'websites':[],'texts':[],'classes':[]}
    root_sites = []
    temp_root_sites = {}
    sites_classes = zip(websites, classes)
    for site,cls in sites_classes:
        group = filter(lambda s: class_order(conversions.page_relation(s[0],True,site),'False') > 0,zip(websites,classes,texts))
        try:
            min_tuple = min(group,key=lambda g:class_ranks[g[1]])
        except:
            import pdb;pdb.set_trace()
        group_class = min_tuple[1]+'Combined'
        group_site = min_tuple[0]
        group_children_text = []
        child_site_texts = zip(child_links, child_text)
        for child,ctext in child_site_texts:
            if conversions.child_or_equal_page(group_site, child, True):
                group_children_text.append(ctext)
                child_idx=child_links.index(child)
                child_links.pop(child_idx)
                child_text.pop(child_idx)
                child_site_texts.pop(child_idx)
        combined_sites['websites'].append(group_site)
        combined_sites['classes'].append(group_class)
        group_text = ' '.join(group_children_text)
        for g in group:
            site_idx = websites.index(g[0])
            websites.pop(site_idx)
            classes.pop(site_idx)
            sites_classes.pop(site_idx)
            texts.pop(site_idx)
        group_text += ' ' + ' '.join(map(lambda g:g[2],group))
        combined_sites['texts'].append(group_text)

    """
        actual_root = conversion.strip_queries(webpage)
        #webpage_no_queries = ul.urlparse.urlparse(webpage)
        #actual_root = webpage_no_queries.scheme + '://' + webpage_no_queries.netloc + webpage_no_queries.path
        webpage_no_queries = re.match(r'(?:www\.)?(?P<content>.+)',webpage_no_queries.netloc + webpage_no_queries.path).groupdict()['content'].rstrip('/')
        child_patt = re.compile(r'^https?://(?:www\.)?{webpage}.+'.format(webpage=webpage_no_queries.lower()))
        if not temp_root_site.has_key(actual_root) or class_order(cls,temp_root_site[actual_root]['class']) > 0:
            temp_root_sites[actual_root] = {'actual_root':actual_root,'child_patt':child_patt,'class':cls+'Combined'})
    for site_dict in temp_root_sites:
        if any(map(lambda rs['child_patt'].match(site_dict['actual_root']),temp_root_sites)):
            continue
        root_sites.append(site_dict)
        combined_sites[site_dict['actual_root']] = {'text':'','class':site_dict['class']}
        for child,ctext in zip(child_links, child_text):
            if site_dict['child_patt'].match(child):


    site_dict = defaultdict(lambda: {'children':set(),'has_parent':False,'text':''})
    for child,ctext in zip(child_links, child_text):
        site_dict[child]['text'] = ctext
    for site,text in zip(websites, texts):
        webpage_no_queries = ul.urlparse.urlparse(webpage)
        webpage_no_queries = re.match(r'(?:www\.)?(?P<content>.+)',webpage_no_queries.netloc + webpage_no_queries.path).groupdict()['content'].rstrip('/')
        child_patt = re.compile(r'^https?://(?:www\.)?{webpage}.+'.format(webpage=webpage_no_queries.lower()))
        site_dict[site]['text'] = text
        for s in websites:
            if s != site and child_patt.match(s):
                site_dict[site]['children'].add(s)
                site_dict[s]['has_parent'] = True
        for c in child_links:
            if c != site and child_patt.match(c):
                site_dict[site]['children'].add(c)
                site_dict[c]['has_parent'] = True
    combined_sites = {'websites':[],'texts':[],'classes':[]}
    def combine_text(site):
        if len(site_dict[site]['children']) > 0:
            ret_text = site_dict[site]['text']
            for c in site_dict[site]['children']:
                ret_text += ' ' + combine_text(c)
            return ret_text
        else:
            return site_dict[site]['text']
    for site,cl in zip(websites, classes):
        if not site_dict[site]['has_parent']:
            combined_sites['websites'].append(site)
            combined_sites['texts'].append(combine_text(site))
            combined_sites['classes'].append(cl+'Combined')
    """
    return combined_sites['websites'], combined_sites['texts'],combined_sites['classes']
def combine_children(websites, texts, classes, child_links, child_text):
    """
    Combines child sites into a group under the main parent site,
    and returns a tuple consisting of the parent site, the parent class,
    and the combined text from all child sites.
    """
    combined_sites = {'websites': [], 'texts': [], 'classes': []}
    # root_sites = []
    # temp_root_sites = {}

    # Create tuples of each search link with its class
    # (i.e. parent, child, identical, no-match)
    sites_classes = zip(websites, classes)

    # Loop over each site and combine
    for site, cls in sites_classes:

        # This creates a group object that contains
        # a list of tuples for every search result
        # which is a child, parent, or equal of another search result
        # for this candidate
        group = filter(
            lambda s: class_order(
                conversions.page_relation(s[0], True, site), 'False'
            ) > 0,
            zip(websites, classes, texts)
        )

        # Get the lowest ranked classes, which at this point should be
        # parents
        try:
            min_tuple = min(group, key=lambda g: class_ranks[g[1]])
        except:
            import pdb
            pdb.set_trace()

        # Setup the '{class}combined' group
        group_class = min_tuple[1]+'Combined'
        group_site = min_tuple[0]
        group_children_text = []

        # Create a tuple of child pages with their text
        child_site_texts = zip(child_links, child_text)

        # Loop over each child site and determine if it is
        # a child of the group_site website.  If so, append its info
        # to the group_children_text list and remove it from the list
        # of child sites
        for child, ctext in child_site_texts:
            if conversions.child_or_equal_page(group_site, child, True):
                group_children_text.append(ctext)
                child_idx = child_links.index(child)
                child_links.pop(child_idx)
                child_text.pop(child_idx)
                child_site_texts.pop(child_idx)

        # Add this site to the combined_sites dict
        combined_sites['websites'].append(group_site)
        combined_sites['classes'].append(group_class)
        group_text = ' '.join(group_children_text)

        # Remove each site in this group from the various lists
        for g in group:
            site_idx = websites.index(g[0])
            websites.pop(site_idx)
            classes.pop(site_idx)
            sites_classes.pop(site_idx)
            texts.pop(site_idx)

        # Add the group's text to to the combined_sites dict
        group_text += ' ' + ' '.join(map(lambda g: g[2], group))
        combined_sites['texts'].append(group_text)

    return (combined_sites['websites'], combined_sites['texts'],
            combined_sites['classes'])
def getlinks(candidate, webpage, state, district_type, district_name):
    """
    Gets all the facebook links found via the Google Search API
    """

    # ### Cleanup input variables

    # District
    district_type = district_type.replace('_', ' ').strip()
    district_type = '+'.join(district_type.split(' '))
    district_name = '+'.join(district_name.strip().split(' '))

    # State
    state = state_map[state.strip()]
    state = '+'.join(state.split(' '))

    # Candidate name
    candidate, last, first = conversions.clean_name(candidate)
    candidate = '+'.join(candidate.split(' '))
    #print 'CANDIDATE: {}'.format(candidate)

    # Setup search urls
    search_urls = []
    extra_children_searches = []
    precise_searches = []

    # Common values
    url = "https://www.googleapis.com/customsearch/v1"
    cx = "011743744063680272768:cp4-iesopjm"
    key = "AIzaSyCdHlGJuMzGBH9hNsEMObffDIkzJ44EQhA"

    search_urls.append(
        u'{url}?cx={cx}&key={key}&hl=en&q={name}+{state}'.format(
            url=url, cx=cx, key=key, name=candidate, state=state
        )
    )

    # Just searches for general about pages
    extra_children_searches.append(
        u'{url}?cx={cx}&key={key}&hl=en&q={name}+{state}+info'.format(
            url=url, cx=cx, key=key, name=candidate, state=state
        )
    )

    # sk=info specifies Facebook's about page
    extra_children_searches.append(
        u'{url}?cx={cx}&key={key}&hl=en&q={name}+{state}+sk=info'.format(
            url=url, cx=cx, key=key, name=candidate, state=state
        )
    )

    precise_searches.append(
        u'{url}?cx={cx}&key={key}&hl=en&q={name}+{state}+campaign'.format(
            url=url, cx=cx, key=key, name=candidate, state=state
        )
    )

    precise_searches.append(
        u'{url}?cx={cx}&key={key}&hl=en&q={name}+{state}+elect'.format(
            url=url, cx=cx, key=key, name=candidate, state=state
        )
    )

    # Clean up encoding of URL's
    search_urls = [
        s.encode(
            chardet.detect(s.encode('utf-8'))['encoding']
        ) for s in search_urls
    ]

    extra_children_searches = [
        s.encode(
            chardet.detect(s.encode('utf-8'))['encoding']
        ) for s in extra_children_searches
    ]

    #print 'SEARCH_URLS: {}'.format(search_urls)

    precise_searches = [
        s.encode(
            chardet.detect(s.encode('utf-8'))['encoding']
        ) for s in precise_searches
    ]

    # This must be  a test for a dummy webside used for testing
    # get_redirect simply gets the final page that returns a 200
    old_webpage = webpage
    if webpage != 'www.gernensamples.com':
        webpage = conversions.get_redirect(webpage)

    #print 'WBBPAGES: {}'.format(webpage)

    has_webpage = True
    #    raise Exception  # why do we need this exception??
    # print 'ok?'
    # Cleanup web pages by removing protocol, subdomain, and trailing '/'

    if has_webpage:
        #print has_webpage
        webpage_stripped = re.match(
            r'(?:https?://)?(?:www\.)?(?P<content>.+)', webpage
        ).groupdict()['content'].rstrip('/')

        old_webpage_stripped = re.match(
            r'(?:https?://)?(?:www\.)?(?P<content>.+)', old_webpage
        ).groupdict()['content'].rstrip('/')

        # TODO strip queries
        webpage_no_queries = ul.urlparse.urlparse(webpage)
        webpage_no_queries = re.match(
            r'(?:www\.)?(?P<content>.+)',
            webpage_no_queries.netloc + webpage_no_queries.path
        ).groupdict()['content'].rstrip('/')

        old_webpage_no_queries = ul.urlparse.urlparse(old_webpage)
        
        #print 'NO:{}'.format(old_webpage_no_queries)
        if old_webpage_no_queries is not None:
            old_webpage_no_queries = re.match(
                r'(?:www\.)?(?P<content>.+)',
                old_webpage_no_queries.netloc + old_webpage_no_queries.path
            ).groupdict()['content'].rstrip('/')

        patt = re.compile(
            r'^https?://(?:www.)?{webpage}/?$'.format(
                webpage=webpage_stripped.lower()
            )
        )
        old_patt = re.compile(
            r'^https?://(?:www.)?{webpage}/?$'.format(
                webpage=old_webpage_stripped.lower()
            )
        )

        child_patt = re.compile(
            r'^https?://(?:www\.)?{webpage}.+'.format(
                webpage=webpage_no_queries.lower()
            )
        )

        old_child_patt = re.compile(
            r'^https?://(?:www\.)?{webpage}.+'.format(
                webpage=old_webpage_no_queries.lower()
            )
        )

    print 'starting'
    n = 4
    while True:
        results = map(lambda x: json.loads(requests.get(x).text), search_urls)
        #for r in results:
        #    print 'error' in r
        if any(map(
                lambda r: ('error' in r and (
                    r['error']['code'] == 403 or r['error']['code'] == 503)
                ), results)):
            print 'sleeping'
            time.sleep(n + random.randint(1, 1000)/1000.)
            n = n*2
        elif any(map(lambda r: 'error' in r, results)):
            raise Exception(', '.join(
                map(
                    lambda r: r['error']['message'],
                    filter(lambda r: 'error' in r, results)
                )
            ))
        else:
            break

    n = 4
    while True:
        child_results = map(
            lambda x: json.loads(requests.get(x).text),
            extra_children_searches
        )
        if any(map(
                lambda r: 'error' in r and (
                    r['error']['code'] == 403 or r['error']['code'] == 503
                ), child_results)):
            print 'sleeping'
            time.sleep(n + random.randint(1, 1000) / 1000.)
            n = n * 2
        elif any(map(
            lambda r: 'error' in r, child_results
        )):
            raise Exception(', '.join(
                map(
                    lambda r: r['error']['message'],
                    filter(lambda r: 'error' in r, child_results)
                )
            ))
        else:
            break

    n = 4
    while True:
        precise_results = map(
            lambda x: json.loads(requests.get(x).text), precise_searches
        )
        if any(map(
            lambda r: 'error' in r and (
                r['error']['code'] == 403 or r['error']['code'] == 503
                ), precise_results)):
            print 'sleeping'
            time.sleep(n + random.randint(1, 1000) / 1000.)
            n = n * 2
        elif any(map(lambda r: 'error' in r, precise_results)):
            raise Exception(', '.join(
                map(
                    lambda r: r['error']['message'],
                    filter(lambda r: 'error' in r, precise_results)
                )
            ))
        else:
            break

  
    if type(results) != list:
        results = [results]

    # Get results from the "items" key and store it in the results variable
    real_results = [
        (r if 'items' in r else {'items': []}) for r in results
    ]
    results = real_results

    # print 'RESULTS:{}'.format(results)
    # Get the result URLs, Extract searchable text from the pagemap
    search_links = [[i['link'].lower() for i in r['items']] for r in results]
    search_text = [
        [u'{title} {link} {pagemap} {snippet}'.format(
            **convert_pagemap_dict(i)
        ).lower().encode('utf-8') for i in r['items']] for r in results
    ]

    # first loop may be unneccessary
    for ri in range(len(search_links)):  # for 1 to number of result objects
        for si in range(len(search_links[ri])):  # for 1 to number of links
            # For each "precise result" (name+state+'elect'),
            # see if the link is equivalent
            # or a sub page of the main results (name+state)
            for r in precise_results:
                if 'items' in r:
                    for i in r['items']:
                        if conversions.child_or_equal_page(
                            search_links[ri][si], i['link'].lower(), True
                        ):
                            search_text[ri][si] += ' bipspecialappearsinprecise'  # noqa

    # Get the result URLs, Extract searchable text from the pagemap
    child_links = [
        i['link'].lower() for r in child_results if 'items' in r
        for i in r['items']
    ]

    child_text = [
        u'{title} {link} {pagemap} {snippet}'.format(
            **convert_pagemap_dict(i)
        ).lower().encode('utf-8') for r in child_results if 'items' in r
        for i in r['items']
    ]

    # Classify each search link based on it's relationship
    # to the provided web page, either PARENT, CHILD, TRUE (identity),
    # or FALSE (no match)
    search_class = [
        map(lambda s: conversions.page_relation(
            s, True, webpage, old_webpage
        ), sl) for sl in search_links
    ]

    # TODO Clean up ssv code

    # Seems to match each search link result against the webpage domain
    ssv = [
        any(map(patt.match, sl)) or any(map(old_patt.match, sl))
        for sl in search_links
    ]

    non_websites = [
        [i['link'] for i in r['items'] if webpage not in i['link']]
        for r in results
    ]

    cs, ct, cc = zip(
        *[combine_children(
            search_links[i], search_text[i], search_class[i],
            child_links, child_text
        ) for i in range(len(search_links))]
    )

    print 'got there', len(results[0]['items'])

    return (non_websites, ssv, webpage_stripped, search_links, search_text,
            [r['items'] for r in results], search_class, cs, ct, cc,
            child_links, child_text)
def getlinks(candidate, webpage, state, district_type, district_name):

    # District
    district_type = district_type.replace('_', ' ').strip()
    district_type = '+'.join(district_type.split(' '))
    district_name = '+'.join(district_name.strip().split(' '))

    # State
    state = state_map[state.strip()]
    state = '+'.join(state.split(' '))

    # Candidate Name
    candidate, last, first = conversions.clean_name(candidate)
    candidate = '+'.join(candidate.split(' '))
    #print candidate

    # Search URLs
    search_urls = []
    precise_searches = []


    url = 'https://www.googleapis.com/customsearch/v1'
    cx = '011743744063680272768:xcugk1a_1t0'
    #cx = '009761440872559920339:eqjjlrdgzma'
    key = 'AIzaSyCdHlGJuMzGBH9hNsEMObffDIkzJ44EQhA&hl'

    # Create search URLs
    search_urls.append(
        u'{url}?cx={cx}&key={key}&hl=en&q={name}+{state}'.format(
            url=url, cx=cx, key=key, name=candidate, state=state
        )
    )

    precise_searches.append(
        u'{url}?cx={cx}&key={key}&hl=en&q={name}+{state}+campaign'.format(
            url=url, cx=cx, key=key, name=candidate, state=state
        )
    )

    precise_searches.append(
        u'{url}?cx={cx}&key={key}&hl=en&q={name}+{state}+elect'.format(
            url=url, cx=cx, key=key, name=candidate, state=state
        )
    )

    # URL Encoding Cleanup
    search_urls = [
        s.encode(
            chardet.detect(s.encode('utf-8'))['encoding']
        ) for s in search_urls
    ]

    precise_searches = [
        s.encode(
            chardet.detect(s.encode('utf-8'))['encoding']
        ) for s in precise_searches
    ]

    # ?? Some sort of test?
    webpage = conversions.twitter_handle_to_web(webpage)
    print webpage
    old_webpage = webpage


    if webpage != 'www.gernensamples.com':
        webpage = conversions.get_redirect(webpage)

    # if webpage == '404' or webpage == 'ERROR':
        # raise Exception

    #print search_urls
    #print precise_searches
    webpage_stripped = re.match(
        r'(?:https?://)?(?:www\.)?(?P<content>.+)', webpage
    ).groupdict()['content'].rstrip('/')

    old_webpage_stripped = re.match(
        r'(?:https?://)?(?:www\.)?(?P<content>.+)', old_webpage
    ).groupdict()['content'].rstrip('/')

    # TODO strip queries
    webpage_no_queries = ul.urlparse.urlparse(webpage)
    webpage_no_queries = re.match(
        r'(?:www\.)?(?P<content>.+)',
        webpage_no_queries.netloc + webpage_no_queries.path
    ).groupdict()['content'].rstrip('/')

    old_webpage_no_queries = ul.urlparse.urlparse(old_webpage)
    old_webpage_no_queries = re.match(
        r'(?:www\.)?(?P<content>.+)',
        old_webpage_no_queries.netloc + old_webpage_no_queries.path
    ).groupdict()['content'].rstrip('/')

    patt = re.compile(
        r'^https?://(?:www.)?{webpage}/?$'.format(
            webpage=webpage_stripped.lower()
        )
    )

    old_patt = re.compile(
        r'^https?://(?:www.)?{webpage}/?$'.format(
            webpage=old_webpage_stripped.lower()
        )
    )

    print 'searching'
    # Timeout work
    n = 4
    while True:
        results = map(lambda x: json.loads(requests.get(x).text), search_urls)
        if any(map(
                lambda r: 'error' in r and (
                    r['error']['code'] == 403 or r['error']['code'] == 503
                ), results)):
            print 'sleeping'
            time.sleep(n + random.randint(1, 1000)/1000.)
            n = n*2
        elif any(map(
                lambda r: 'error' in r, results)):
            raise Exception(', '.join(
                map(lambda r: r['error']['message'],
                    filter(lambda r: 'error' in r, results))
            ))
        else:
            break

    n = 4
    while True:
        precise_results = map(
            lambda x: json.loads(requests.get(x).text), precise_searches
        )
        if any(map(
                lambda r: 'error' in r and (
                    r['error']['code'] == 403 or r['error']['code'] == 503
                ), precise_results)):
            print 'sleeping'
            time.sleep(n + random.randint(1, 1000)/1000.)
            n = n*2
        elif any(map(lambda r: 'error' in r, precise_results)):
            raise Exception(', '.join(
                map(lambda r: r['error']['message'],
                    filter(lambda r: 'error' in r, precise_results))
            ))
        else:
            break
    print 'done searching'

    if type(results) != list:
        results = [results]

    # Get results
    real_results = [(r if 'items' in r else {'items': []}) for r in results]
    results = real_results

    search_links = [
        [conversions.clean_twitter(i['link'].lower()) for i in r['items']]
        for r in results
    ]

    search_text = [
        [u'{title} {link} {pagemap} {snippet}'.format(
            **convert_pagemap_dict(i)
        ).lower().encode('utf-8') for i in r['items']] for r in results
    ]

    for ri in range(len(search_links)):
        for si in range(len(search_links[ri])):
            for r in precise_results:
                if 'items' in r:
                    for i in r['items']:
                        if conversions.child_or_equal_page(
                                search_links[ri][si],
                                conversions.clean_twitter(i['link'].lower()),
                                True):
                            search_text[ri][si] += ' bipspecialappearsinprecise'

    child_links = []
    child_text = []
    search_class = [map(lambda s: conversions.page_relation(
        s, False, webpage, old_webpage), sl) for sl in search_links
    ]

    # TODO Clean up ssv code
    ssv = [
        any(map(patt.match, sl)) or any(map(old_patt.match, sl))
        for sl in search_links
    ]

    non_websites = [
        [
            i['link'] for i in r['items'] if webpage not in i['link']
        ] for r in results
    ]

    cs, ct, cc = zip(*[combine_children(
        search_links[i],
        search_text[i],
        search_class[i],
        child_links,
        child_text) for i in range(len(search_links))
        ]
    )

    print 'got there', len(results[0]['items'])
    return (non_websites, ssv, webpage_stripped, search_links, search_text,
            [r['items'] for r in results], search_class,
            cs, ct, cc, child_links, child_text)
Exemple #7
0
def combine_children(websites, texts, classes, child_links, child_text):
    combined_sites = {'websites': [], 'texts': [], 'classes': []}
    root_sites = []
    temp_root_sites = {}
    sites_classes = zip(websites, classes)
    for site, cls in sites_classes:
        group = filter(
            lambda s: class_order(conversions.page_relation(s[0], True, site),
                                  'False') > 0, zip(websites, classes, texts))
        try:
            min_tuple = min(group, key=lambda g: class_ranks[g[1]])
        except:
            import pdb
            pdb.set_trace()
        group_class = min_tuple[1] + 'Combined'
        group_site = min_tuple[0]
        group_children_text = []
        child_site_texts = zip(child_links, child_text)
        for child, ctext in child_site_texts:
            if conversions.child_or_equal_page(group_site, child, True):
                group_children_text.append(ctext)
                child_idx = child_links.index(child)
                child_links.pop(child_idx)
                child_text.pop(child_idx)
                child_site_texts.pop(child_idx)
        combined_sites['websites'].append(group_site)
        combined_sites['classes'].append(group_class)
        group_text = ' '.join(group_children_text)
        for g in group:
            site_idx = websites.index(g[0])
            websites.pop(site_idx)
            classes.pop(site_idx)
            sites_classes.pop(site_idx)
            texts.pop(site_idx)
        group_text += ' ' + ' '.join(map(lambda g: g[2], group))
        combined_sites['texts'].append(group_text)
    """
        actual_root = conversion.strip_queries(webpage)
        #webpage_no_queries = ul.urlparse.urlparse(webpage)
        #actual_root = webpage_no_queries.scheme + '://' + webpage_no_queries.netloc + webpage_no_queries.path
        webpage_no_queries = re.match(r'(?:www\.)?(?P<content>.+)',webpage_no_queries.netloc + webpage_no_queries.path).groupdict()['content'].rstrip('/')
        child_patt = re.compile(r'^https?://(?:www\.)?{webpage}.+'.format(webpage=webpage_no_queries.lower()))
        if not temp_root_site.has_key(actual_root) or class_order(cls,temp_root_site[actual_root]['class']) > 0:
            temp_root_sites[actual_root] = {'actual_root':actual_root,'child_patt':child_patt,'class':cls+'Combined'})
    for site_dict in temp_root_sites:
        if any(map(lambda rs['child_patt'].match(site_dict['actual_root']),temp_root_sites)):
            continue
        root_sites.append(site_dict)
        combined_sites[site_dict['actual_root']] = {'text':'','class':site_dict['class']}
        for child,ctext in zip(child_links, child_text):
            if site_dict['child_patt'].match(child):


    site_dict = defaultdict(lambda: {'children':set(),'has_parent':False,'text':''})
    for child,ctext in zip(child_links, child_text):
        site_dict[child]['text'] = ctext
    for site,text in zip(websites, texts):
        webpage_no_queries = ul.urlparse.urlparse(webpage)
        webpage_no_queries = re.match(r'(?:www\.)?(?P<content>.+)',webpage_no_queries.netloc + webpage_no_queries.path).groupdict()['content'].rstrip('/')
        child_patt = re.compile(r'^https?://(?:www\.)?{webpage}.+'.format(webpage=webpage_no_queries.lower()))
        site_dict[site]['text'] = text
        for s in websites:
            if s != site and child_patt.match(s):
                site_dict[site]['children'].add(s)
                site_dict[s]['has_parent'] = True
        for c in child_links:
            if c != site and child_patt.match(c):
                site_dict[site]['children'].add(c)
                site_dict[c]['has_parent'] = True
    combined_sites = {'websites':[],'texts':[],'classes':[]}
    def combine_text(site):
        if len(site_dict[site]['children']) > 0:
            ret_text = site_dict[site]['text']
            for c in site_dict[site]['children']:
                ret_text += ' ' + combine_text(c)
            return ret_text
        else:
            return site_dict[site]['text']
    for site,cl in zip(websites, classes):
        if not site_dict[site]['has_parent']:
            combined_sites['websites'].append(site)
            combined_sites['texts'].append(combine_text(site))
            combined_sites['classes'].append(cl+'Combined')
    """
    return combined_sites['websites'], combined_sites['texts'], combined_sites[
        'classes']
Exemple #8
0
def getlinks(candidate, webpage, state, district_type, district_name):
    district_type = district_type.replace('_', ' ').strip()
    state = state_map[state.strip()]
    candidate, last, first = conversions.clean_name(candidate)
    candidate = '+'.join(candidate.split(' '))
    print candidate
    state = '+'.join(state.split(' '))
    district_type = '+'.join(district_type.split(' '))
    district_name = '+'.join(district_name.strip().split(' '))
    search_urls = []
    extra_children_searches = []
    precise_searches = []
    search_urls.append(
        u'https://www.googleapis.com/customsearch/v1?cx=011743744063680272768:cp4-iesopjm&key=AIzaSyCdHlGJuMzGBH9hNsEMObffDIkzJ44EQhA&hl=en&q={name}+{state}'
        .format(name=candidate, state=state))
    extra_children_searches.append(
        u'https://www.googleapis.com/customsearch/v1?cx=011743744063680272768:cp4-iesopjm&key=AIzaSyCdHlGJuMzGBH9hNsEMObffDIkzJ44EQhA&hl=en&q={name}+{state}+info'
        .format(name=candidate, state=state))
    extra_children_searches.append(
        u'https://www.googleapis.com/customsearch/v1?cx=011743744063680272768:cp4-iesopjm&key=AIzaSyCdHlGJuMzGBH9hNsEMObffDIkzJ44EQhA&hl=en&q={name}+{state}+sk=info'
        .format(name=candidate, state=state))
    precise_searches.append(
        u'https://www.googleapis.com/customsearch/v1?cx=011743744063680272768:cp4-iesopjm&key=AIzaSyCdHlGJuMzGBH9hNsEMObffDIkzJ44EQhA&hl=en&q={name}+{state}+campaign'
        .format(name=candidate, state=state))
    precise_searches.append(
        u'https://www.googleapis.com/customsearch/v1?cx=011743744063680272768:cp4-iesopjm&key=AIzaSyCdHlGJuMzGBH9hNsEMObffDIkzJ44EQhA&hl=en&q={name}+{state}+elect'
        .format(name=candidate, state=state))
    search_urls = [
        s.encode(chardet.detect(s.encode('utf-8'))['encoding'])
        for s in search_urls
    ]
    extra_children_searches = [
        s.encode(chardet.detect(s.encode('utf-8'))['encoding'])
        for s in extra_children_searches
    ]
    precise_searches = [
        s.encode(chardet.detect(s.encode('utf-8'))['encoding'])
        for s in precise_searches
    ]
    old_webpage = webpage
    if webpage != 'www.gernensamples.com':
        webpage = conversions.get_redirect(webpage)
    #if webpage == '404' or webpage == 'ERROR':
    #raise Exception
    websites = []
    webpage_stripped = re.match(r'(?:https?://)?(?:www\.)?(?P<content>.+)',
                                webpage).groupdict()['content'].rstrip('/')
    old_webpage_stripped = re.match(
        r'(?:https?://)?(?:www\.)?(?P<content>.+)',
        old_webpage).groupdict()['content'].rstrip('/')
    #TODO strip queries
    webpage_no_queries = ul.urlparse.urlparse(webpage)
    webpage_no_queries = re.match(
        r'(?:www\.)?(?P<content>.+)', webpage_no_queries.netloc +
        webpage_no_queries.path).groupdict()['content'].rstrip('/')
    old_webpage_no_queries = ul.urlparse.urlparse(old_webpage)
    old_webpage_no_queries = re.match(
        r'(?:www\.)?(?P<content>.+)', old_webpage_no_queries.netloc +
        old_webpage_no_queries.path).groupdict()['content'].rstrip('/')
    patt = re.compile(r'^https?://(?:www.)?{webpage}/?$'.format(
        webpage=webpage_stripped.lower()))
    old_patt = re.compile(r'^https?://(?:www.)?{webpage}/?$'.format(
        webpage=old_webpage_stripped.lower()))
    child_patt = re.compile(r'^https?://(?:www\.)?{webpage}.+'.format(
        webpage=webpage_no_queries.lower()))
    old_child_patt = re.compile(r'^https?://(?:www\.)?{webpage}.+'.format(
        webpage=old_webpage_no_queries.lower()))
    n = 4
    while True:
        results = map(lambda x: json.loads(requests.get(x).text), search_urls)
        if any(
                map(
                    lambda r: r.has_key('error') and
                    (r['error']['code'] == 403 or r['error']['code'] == 503),
                    results)):
            print 'sleeping'
            time.sleep(n + random.randint(1, 1000) / 1000.)
            n = n * 2
        elif any(map(lambda r: r.has_key('error'), results)):
            raise Exception(', '.join(
                map(lambda r: r['error']['message'],
                    filter(lambda r: r.has_key('error'), results))))
        else:
            break
    n = 4
    while True:
        child_results = map(lambda x: json.loads(requests.get(x).text),
                            extra_children_searches)
        if any(
                map(
                    lambda r: r.has_key('error') and
                    (r['error']['code'] == 403 or r['error']['code'] == 503),
                    child_results)):
            print 'sleeping'
            time.sleep(n + random.randint(1, 1000) / 1000.)
            n = n * 2
        elif any(map(lambda r: r.has_key('error'), child_results)):
            raise Exception(', '.join(
                map(lambda r: r['error']['message'],
                    filter(lambda r: r.has_key('error'), child_results))))
        else:
            break
    n = 4
    while True:
        precise_results = map(lambda x: json.loads(requests.get(x).text),
                              precise_searches)
        if any(
                map(
                    lambda r: r.has_key('error') and
                    (r['error']['code'] == 403 or r['error']['code'] == 503),
                    precise_results)):
            print 'sleeping'
            time.sleep(n + random.randint(1, 1000) / 1000.)
            n = n * 2
        elif any(map(lambda r: r.has_key('error'), precise_results)):
            raise Exception(', '.join(
                map(lambda r: r['error']['message'],
                    filter(lambda r: r.has_key('error'), precise_results))))
        else:
            break

    if type(results) != list:
        print type(results)
        results = [results]
    real_results = [(r if r.has_key('items') else {
        'items': []
    }) for r in results]
    results = real_results
    search_links = [[i['link'].lower() for i in r['items']] for r in results]
    search_text = [[
        u'{title} {link} {pagemap} {snippet}'.format(
            **convert_pagemap_dict(i)).lower().encode('utf-8')
        for i in r['items']
    ] for r in results]
    for ri in range(len(search_links)):
        for si in range(len(search_links[ri])):
            for r in precise_results:
                if r.has_key('items'):
                    for i in r['items']:
                        if conversions.child_or_equal_page(
                                search_links[ri][si], i['link'].lower(), True):
                            search_text[ri][
                                si] += ' bipspecialappearsinprecise'
    child_links = [
        i['link'].lower() for r in child_results if r.has_key('items')
        for i in r['items']
    ]
    child_text = [
        u'{title} {link} {pagemap} {snippet}'.format(
            **convert_pagemap_dict(i)).lower().encode('utf-8')
        for r in child_results if r.has_key('items') for i in r['items']
    ]
    #search_text = [[u'{title} {link} {pagemap} {snippet}'.format(**i).lower().encode('utf-8') for i in r['items']] for r in results]
    search_class = [
        map(lambda s: conversions.page_relation(s, True, webpage, old_webpage),
            sl) for sl in search_links
    ]
    #search_class = [map(lambda s: 'True' if patt.match(s) != None or old_patt.match(s) != None else ('Child' if child_patt.match(s) != None or old_child_patt.match(s) != None else 'False'),sl) for sl in search_links]
    #print search_text
    #TODO Clean up ssv code
    ssv = [
        any(map(patt.match, sl)) or any(map(old_patt.match, sl))
        for sl in search_links
    ]
    non_websites = [[
        i['link'] for i in r['items'] if webpage not in i['link']
    ] for r in results]
    cs, ct, cc = zip(*[
        combine_children(search_links[i], search_text[i], search_class[i],
                         child_links, child_text)
        for i in range(len(search_links))
    ])
    print 'got there', len(results[0]['items'])
    return non_websites, ssv, webpage_stripped, search_links, search_text, [
        r['items'] for r in results
    ], search_class, cs, ct, cc, child_links, child_text