Exemplos de DataManager.get_project_source em Python

Exemplo n.º 1

Arquivo: scraper.py Projeto: bobflagg/tme

def collect_source_data():
    dbm = DataManager()
    for pid, source_url in dbm.get_project_source():
        print "working on project %s" % pid
        print "url: %s" % source_url
        try:
            r = requests.get(source_url)
            fp = codecs_open("%s/html/%s.html" % (HOME_DIR, pid), "w", "utf-8" )
            fp.write(r.text)
            fp.close()
            sleep(1)
        except ConnectionError, e:
            print "failed"
            print e.message