Python extract_hrefs Examples

Example #1

File: gevent_dqp_demo.py Project: reorx/learn_spider

def fetch_and_extract(url, data_queue):
    r = urllib2.urlopen(url)
    html = r.read()

    hrefs = demo_helpers.extract_hrefs(html)
    #print('fetch ', url, html, hrefs)
    data_queue.put_nowait((url, hrefs))

Example #2

File: gevent_tq_dq_demo.py Project: reorx/learn_spider

def url_worker(name, tq, dq):
    print('worker start', name)
    while True:
        url = tq.get()
        print('worker {} get {}'.format(name, url))

        r = urllib2.urlopen(url)
        html = r.read()

        hrefs = demo_helpers.extract_hrefs(html)

        dq.put_nowait((url, hrefs))

Example #3

File: gevent_tq_demo.py Project: reorx/learn_spider

def url_worker(name, processed_urls, add_to_all, q):
    print("worker start", name)
    while True:
        url = q.get()
        print("worker {} get {}".format(name, url))

        r = urllib2.urlopen(url)
        html = r.read()

        hrefs = demo_helpers.extract_hrefs(html)
        # print('fetch ', url, html, hrefs)
        for sub_url in hrefs:
            add_to_all(sub_url)
            if sub_url not in processed_urls:
                q.put_nowait(sub_url)

        if url in processed_urls:
            print("Duplicate processed url {}".format(url))
        else:
            processed_urls.add(url)
        q.task_done()