from crawler import Crawler
from bs4 import BeautifulSoup
crawler = Crawler()
# returns a list of the available forms
forms = crawler.get_forms('https://github.com/login')
# submits the login form
crawler.submit(forms[1], {'login': '******', 'password': '******'})
# just a simple GET request, returns a response object
response = crawler.visit('https://github.com')
# lets use BeautifulSoup to parse the response text
html = BeautifulSoup(response.text, 'lxml')
# get a list of our repositories by scraping the html
for repo in html.find(id='repo_listing').find_all('span', {'class': 'repo'}):
print repo.text
print '\n', crawler.get_cookies()
print '\n', crawler.get_headers()
