def handle(self, *args, **kwargs):


        for index in [0,1,2,3,4,5,6,7]:
            scraper = CongressLeadership(index)
            if index == 0:
                scraper.get_speaker_links(scraper.leaders[index])
            elif index == 1:
                scraper.get_house_majority_leader_links(scraper.leaders[index])
            elif index == 2:
                scraper.get_house_minority_leader_links(scraper.leaders[index])
            elif index == 3:
                scraper.get_house_minority_whip_links(scraper.leaders[index])
            elif index == 4:
                scraper.get_house_majority_whip_links(scraper.leaders[index])
            elif index == 5:
                scraper.get_senate_majority_leader_links(scraper.leaders[index])
            elif index == 6:
                scraper.get_senate_minority_leader_links(scraper.leaders[index])
            elif index == 7:
                scraper.get_white_house_links(scraper.leaders[index])

            for link in scraper.links:
                
                #store in database 
                #leave sources as get or create so we can keep track of when page urls change or new members are added easily
                (source,created) = Source.objects.get_or_create(url=scraper.urls[index], source_type=3)

                try:
                    doc = scraper.extract(link)
              
                    try:
                        release = Release.objects.get(url=link)
                        release.source=source
                        release.title=doc['title']
                        release.body=doc['text']
                        release.date=doc['date']
                        release.save()
                        print "saved release %s" % link
                    except Release.DoesNotExist:
                        release = Release(url=link,
                                          title= doc['title'],
                                          body=doc['text'],
                                          date=doc['date'],
                                          source=source)
                        release.save()
                        print "saved release %s" % link

                except Exception as e:
                    print e
                    buf = StringIO()
                    print_exc(1000, buf)
                    f = SourceScrapeFailure.objects.create(source=source, 
                                                           traceback=buf.getvalue(), 
                                                           description=unicode(e))
def set_release(apps, schema_editor):
    DocumentRelease = apps.get_model('docs', 'DocumentRelease')
    Release = apps.get_model('releases', 'Release')
    for dr in DocumentRelease.objects.exclude(version='dev'):
        try:
            release = Release.objects.get(version=dr.version)
        except Release.DoesNotExist:
            # Hack: we need the actual Release model, specifically
            # its overridden save(), to create a release.
            from releases.models import Release as RealRelease
            release = RealRelease(version=dr.version, date=None)
            release.save()
        # Because of the hack above, we cannot use dr.release = release.
        dr.release_id = release.pk
        dr.save()
Example #3
0
 def init_project(self, site: Site, project):
     commits = gl.get_commits(project)
     for commit in commits:
         Release(site=site, version=commit.id, message=commit.message, committed_time=commit.committed_date).save()
     gl.add_hook(project, "http://{}:{}{}".format(SERVER_NAME if SERVER_NAME else SERVER_IP, SERVER_PORT, reverse('hook-list')))
     site.status = 1
     site.save()
def set_release(apps, schema_editor):
    DocumentRelease = apps.get_model("docs", "DocumentRelease")
    Release = apps.get_model("releases", "Release")
    for dr in DocumentRelease.objects.exclude(version="dev"):
        try:
            release = Release.objects.get(version=dr.version)
        except Release.DoesNotExist:
            # Hack: we need the actual Release model, specifically
            # its overridden save(), to create a release.
            from releases.models import Release as RealRelease

            release = RealRelease(version=dr.version, date=None)
            release.save()
        # Because of the hack above, we cannot use dr.release = release.
        dr.release_id = release.pk
        dr.save()
Example #5
0
 def flush_commits(self, site: Site, commits):
     for commit in commits:
         Release(site=site, version=commit["id"], message=commit["message"], committed_time=commit["timestamp"]).save()