예제 #1
0
파일: exline.py 프로젝트: attomos/Oppomus
def crawl():
    #Use logger to debug the code
    Logger.debug('Hello google')
    #Get the html knowledge by parsing url
    soup = get_b_soup('http://www.google.com')
    #Send the data to output pipe line
    Queues.send_to_output(soup.head.title.text)
    Queues.send_to_output('http://www.google.com' + soup.head.meta['content'])
예제 #2
0
파일: exline.py 프로젝트: opplieam/Oppomus
def crawl():
    #Use logger to debug the code
    Logger.debug('Hello google')
    #Get the html knowledge by parsing url
    soup = get_b_soup('http://www.google.com')
    #Send the data to output pipe line
    Queues.send_to_output(soup.head.title.text)
    Queues.send_to_output('http://www.google.com' + soup.head.meta['content'])
예제 #3
0
def collect(url_queue):
    #Use logger to debug the code
    Logger.debug('Hello amazon')
    #Get the html knowledge by parsing url
    soup = get_b_soup(base_path)
    #Travel with html knowledge
    for all_url in soup.findAll('a', href=re.compile('/gp/product/')):
        #REQUIRED
        #Send all url via url_queue
        url_queue.put(base_path + all_url['href'])
예제 #4
0
파일: nested.py 프로젝트: attomos/Oppomus
def crawl_thread(url, module):
    Logger.debug('Thread start')
    encoding = getattr(module, 'ENCODING', None)
    try:
        soup = get_b_soup(url, encoding=encoding)
        module.crawl(soup)
    except:
        Logger.error('Crawl error url: ' + url)
        Logger.error(traceback.format_exc())
        return
    Logger.debug('Thread done')
예제 #5
0
파일: nested.py 프로젝트: opplieam/Oppomus
def crawl_thread(url, module):
    Logger.debug('Thread start')
    encoding = getattr(module, 'ENCODING', None)
    try:
        soup = get_b_soup(url, encoding=encoding)
        module.crawl(soup)
    except:
        Logger.error('Crawl error url: ' + url)
        Logger.error(traceback.format_exc())
        return
    Logger.debug('Thread done')