Esempi in Python per downloadUrl

Esempio n. 1

File: test3_processor.py Progetto: dustin-stoltz/snowcrawl

def downloadAndClassifyUrl( (url, params) ):
	(text, download_stats) = snowcrawl.downloadUrl( 'http://'+url )
	(kept, classify_stats) = classifyText( text, params )
	(edges, edge_stats) = snowcrawl.findEdges( url, text )
	if kept:
		return (1, text, edges, edges, download_stats + classify_stats + edge_stats)
	else:
		return (0, text, [], edges, download_stats + classify_stats + edge_stats)

Esempio n. 2

File: download_blog_front_pages.py Progetto: danithaca/sugarcane

def myUrlProcesser( (url, params) ):
    (text, download_stats) = snowcrawl.downloadUrl( 'http://'+url )
    (kept, classify_stats) = (0, [])
    (edges, edge_stats) = snowcrawl.findEdges( url, text )
    if kept:
        return (kept, text, edges, edges, download_stats + classify_stats + edge_stats)
    else:
        return (kept, text, [], edges, download_stats + classify_stats + edge_stats)