def handle(self, *args, **kwargs):
for index in [0,1,2,3,4,5,6,7]:
scraper = CongressLeadership(index)
if index == 0:
scraper.get_speaker_links(scraper.leaders[index])
elif index == 1:
scraper.get_house_majority_leader_links(scraper.leaders[index])
elif index == 2:
scraper.get_house_minority_leader_links(scraper.leaders[index])
elif index == 3:
scraper.get_house_minority_whip_links(scraper.leaders[index])
elif index == 4:
scraper.get_house_majority_whip_links(scraper.leaders[index])
elif index == 5:
scraper.get_senate_majority_leader_links(scraper.leaders[index])
elif index == 6:
scraper.get_senate_minority_leader_links(scraper.leaders[index])
elif index == 7:
scraper.get_white_house_links(scraper.leaders[index])
for link in scraper.links:
#store in database
#leave sources as get or create so we can keep track of when page urls change or new members are added easily
(source,created) = Source.objects.get_or_create(url=scraper.urls[index], source_type=3)
try:
doc = scraper.extract(link)
try:
release = Release.objects.get(url=link)
release.source=source
release.title=doc['title']
release.body=doc['text']
release.date=doc['date']
release.save()
print "saved release %s" % link
except Release.DoesNotExist:
release = Release(url=link,
title= doc['title'],
body=doc['text'],
date=doc['date'],
source=source)
release.save()
print "saved release %s" % link
except Exception as e:
print e
buf = StringIO()
print_exc(1000, buf)
f = SourceScrapeFailure.objects.create(source=source,
traceback=buf.getvalue(),
description=unicode(e))
def set_release(apps, schema_editor):
DocumentRelease = apps.get_model('docs', 'DocumentRelease')
Release = apps.get_model('releases', 'Release')
for dr in DocumentRelease.objects.exclude(version='dev'):
try:
release = Release.objects.get(version=dr.version)
except Release.DoesNotExist:
# Hack: we need the actual Release model, specifically
# its overridden save(), to create a release.
from releases.models import Release as RealRelease
release = RealRelease(version=dr.version, date=None)
release.save()
# Because of the hack above, we cannot use dr.release = release.
dr.release_id = release.pk
dr.save()
def init_project(self, site: Site, project):
commits = gl.get_commits(project)
for commit in commits:
Release(site=site, version=commit.id, message=commit.message, committed_time=commit.committed_date).save()
gl.add_hook(project, "http://{}:{}{}".format(SERVER_NAME if SERVER_NAME else SERVER_IP, SERVER_PORT, reverse('hook-list')))
site.status = 1
site.save()
def set_release(apps, schema_editor):
DocumentRelease = apps.get_model("docs", "DocumentRelease")
Release = apps.get_model("releases", "Release")
for dr in DocumentRelease.objects.exclude(version="dev"):
try:
release = Release.objects.get(version=dr.version)
except Release.DoesNotExist:
# Hack: we need the actual Release model, specifically
# its overridden save(), to create a release.
from releases.models import Release as RealRelease
release = RealRelease(version=dr.version, date=None)
release.save()
# Because of the hack above, we cannot use dr.release = release.
dr.release_id = release.pk
dr.save()
def flush_commits(self, site: Site, commits):
for commit in commits:
Release(site=site, version=commit["id"], message=commit["message"], committed_time=commit["timestamp"]).save()