def scrape_reports_and_decisions(since=None, page=None, everything=False,
coder_type=settings.GEOCODER):
logger = scrape_reports_and_decisions.get_logger()
if coder_type == "google":
geocoder = gmaps.GoogleGeocoder(settings.GOOGLE_API_KEY)
geocoder.bounds = settings.GEO_BOUNDS
geocoder.region = settings.GEO_REGION
else:
geocoder = arcgis.ArcGISCoder(settings.ARCGIS_CLIENT_ID,
settings.ARCGIS_CLIENT_SECRET)
if page is not None:
proposals_json = scrape.get_proposals_for_page(page, geocoder)
else:
if not since:
# If there was no last run, the scraper will fetch all
# proposals.
since = last_run()
proposals_json = scrape.get_proposals_since(dt=since, geocoder=geocoder)
proposals = []
for p_dict in proposals_json:
p = create_proposal_from_json(p_dict)
if p:
p.save()
proposals.append(p)
else:
logger.error("Could not create proposal from dictionary:",
p_dict)
def scrape_reports_and_decisions(since=None,
page=None,
coder_type=settings.GEOCODER):
"""
Task that scrapes the reports and decisions page
"""
if coder_type == "google":
geocoder = gmaps.GoogleGeocoder(settings.GOOGLE_API_KEY)
geocoder.bounds = settings.GEO_BOUNDS
geocoder.region = settings.GEO_REGION
else:
geocoder = arcgis.ArcGISCoder(settings.ARCGIS_CLIENT_ID,
settings.ARCGIS_CLIENT_SECRET)
if page is not None:
proposals_json = scrape.get_proposals_for_page(page, geocoder)
else:
if not since:
# If there was no last run, the scraper will fetch all
# proposals.
since = last_run()
if not since:
latest_proposal = Proposal.objects.latest()
if latest_proposal:
since = latest_proposal.updated
if not since:
# If there is no record of a previous run, fetch
# proposals posted since the previous Monday.
now = datetime.now().replace(hour=0,
minute=0,
second=0,
microsecond=0)
since = now - timedelta(days=7 + now.weekday())
proposals_json = scrape.get_proposals_since(dt=since,
geocoder=geocoder)
proposals = []
for p_dict in proposals_json:
p = create_proposal_from_json(p_dict)
if p:
p.save()
proposals.append(p)
else:
logger.error("Could not create proposal from dictionary:", p_dict)
return proposals
def scrape_reports_and_decisions(since=None):
if not since:
# If there was no last run, the scraper will fetch all
# proposals.
since = last_run()
# Array of dicts:
proposals_json = scrape.get_proposals_since(since)
proposals = []
for p_dict in proposals_json:
p = create_proposal_from_json(p_dict)
p.save()
proposals.append(p)
def scrape_reports_and_decisions(since=None):
if not since:
# If there was no last run, the scraper will fetch all
# proposals.
since = last_run()
# Array of dicts:
proposals_json = scrape.get_proposals_since(since)
proposals = []
for p_dict in proposals_json:
p = create_proposal_from_json(p_dict)
p.save()
proposals.append(p)
def scrape_reports_and_decisions(since=None, page=None,
coder_type=settings.GEOCODER):
"""
Task that scrapes the reports and decisions page
"""
if coder_type == "google":
geocoder = gmaps.GoogleGeocoder(settings.GOOGLE_API_KEY)
geocoder.bounds = settings.GEO_BOUNDS
geocoder.region = settings.GEO_REGION
else:
geocoder = arcgis.ArcGISCoder(settings.ARCGIS_CLIENT_ID,
settings.ARCGIS_CLIENT_SECRET)
if page is not None:
proposals_json = scrape.get_proposals_for_page(page, geocoder)
else:
if not since:
# If there was no last run, the scraper will fetch all
# proposals.
since = last_run()
if not since:
latest_proposal = Proposal.objects.latest()
if latest_proposal:
since = latest_proposal.updated
if not since:
# If there is no record of a previous run, fetch
# proposals posted since the previous Monday.
now = datetime.now().replace(hour=0, minute=0,
second=0, microsecond=0)
since = now - timedelta(days=7 + now.weekday())
proposals_json = scrape.get_proposals_since(dt=since, geocoder=geocoder)
proposals = []
for p_dict in proposals_json:
p = create_proposal_from_json(p_dict)
if p:
p.save()
proposals.append(p)
else:
logger.error("Could not create proposal from dictionary:",
p_dict)
return proposals