def export_graph_data():
    data_file = open(EXPORT_EDGES_FILE, "w")
    db = db_utils.getDBInstance()

    nodes = set([])
    data_file.write("Source\tTarget\n")
 
    pages = db.webpages.find()
    for page in pages:
        ilinks = page["incoming_links"]
        for link in ilinks:
            if link.startswith("javascript"):
                continue
            if page["url"].startswith("javascript"):
                continue
            link = link.replace(",", "")
            nodes.add(link)
            page["url"] = page["url"].replace(",", "")
            nodes.add(page["url"])
            data_file.write(("%s\t%s\n"%(link, page["url"])))



    nodes_file = open(EXPORT_NODES_FILE, "w")    

    nodes_file.write("Id\tLabel\n")
    for node in nodes:
        nodes_file.write("%s\t%s\n"%(node, utils.domainOf(node)))
    
    nodes_file.close() 
    data_file.close()
def crawl_manager(number_of_processes=12 ):
    db = db_utils.getDBInstance()
    seedStartURLsInDB(db, START_URLS)
    number_of_iterations = 10 
    while number_of_iterations:
        pool = Pool(number_of_processes)
        urls = getUnvisitedURLsFromDB(db, number_of_processes)
        result = pool.map(crawler.crawl_url, urls)
        
        number_of_iterations -= 1
def draw_graph3d(graph_colormap='winter', bgcolor = (1, 1, 1),
                 node_size=0.03,
                 edge_color=(0.8, 0.8, 0.8), edge_size=0.002,
                 text_size=0.008, text_color=(0, 0, 0)): 
    
    H = nx.Graph()
    
    db = db_utils.getDBInstance()
    nodes = db.webpages.find()
    edges = []
    for node in nodes:
        iurls = node["incoming_links"]
        for iurl in iurls:
            H.add_edge(iurl, node["url"])

    G= nx.convert_node_labels_to_integers(H)

    graph_pos=nx.spring_layout(G, dim=3)

    # numpy array of x,y,z positions in sorted node order
    xyz=np.array([graph_pos[v] for v in sorted(G)])

    # scalar colors
    scalars=np.array(G.nodes())+5
    mlab.figure(1, bgcolor=bgcolor)
    mlab.clf()

    pts = mlab.points3d(xyz[:,0], xyz[:,1], xyz[:,2],
                        scalars,
                        scale_factor=node_size,
                        scale_mode='none',
                        colormap=graph_colormap,
                        resolution=20)

    for i, (x, y, z) in enumerate(xyz):
        label = mlab.text(x, y, str(i), z=z,
                          width=text_size, name=str(i), color=text_color)
        label.property.shadow = True

    pts.mlab_source.dataset.lines = np.array(G.edges())
    tube = mlab.pipeline.tube(pts, tube_radius=edge_size)
    mlab.pipeline.surface(tube, color=edge_color)

    mlab.show() # interactive window
def crawl_url(url, headless=True, save_into_db=True):
    print "Crawling URL",url
    iurl_hash = utils.get_url_hash(url)
    update = {iurl_hash: url}

    db = db_utils.getDBInstance()  
    if regex_domain_match(db, url):
        print "Skipping: ",url
        
    db.crawl_queue.remove(update)
    url = utils.sanitize_url(url)
    url_hash = utils.get_url_hash(url)
    db_query = {'url_hash': url_hash}

    if headless:
        display = Display(visible=0, size=(800, 600))
        display.start()

    
    obj_in_db = db.webpages.find_one(db_query)
    
    webpage = None
    if not obj_in_db:
        webpage = WebPage(url)


    browser = webdriver.Chrome(desired_capabilities=capabilities)
    browser.set_page_load_timeout(30)
    #browser.implicitly_wait(5)
    try:
        print "Visiting page: ",url
        if not url.startswith("http"):
            raise Exception

        browser.get(url)
        #time.sleep(1)
    except Exception, e:
        print "Error Occured"
        browser.quit()
        print e
        return -1