コード例 #1
0
def scrape_reports_and_decisions(since=None, page=None, everything=False,
                                 coder_type=settings.GEOCODER):
    logger = scrape_reports_and_decisions.get_logger()

    if coder_type == "google":
        geocoder = gmaps.GoogleGeocoder(settings.GOOGLE_API_KEY)
        geocoder.bounds = settings.GEO_BOUNDS
        geocoder.region = settings.GEO_REGION
    else:
        geocoder = arcgis.ArcGISCoder(settings.ARCGIS_CLIENT_ID,
                                      settings.ARCGIS_CLIENT_SECRET)

    if page is not None:
        proposals_json = scrape.get_proposals_for_page(page, geocoder)
    else:
        if not since:
            # If there was no last run, the scraper will fetch all
            # proposals.
            since = last_run()
        proposals_json = scrape.get_proposals_since(dt=since, geocoder=geocoder)

    proposals = []

    for p_dict in proposals_json:
        p = create_proposal_from_json(p_dict)

        if p:
            p.save()
            proposals.append(p)
        else:
            logger.error("Could not create proposal from dictionary:",
                         p_dict)
コード例 #2
0
def scrape_reports_and_decisions(since=None,
                                 page=None,
                                 coder_type=settings.GEOCODER):
    """
    Task that scrapes the reports and decisions page
    """
    if coder_type == "google":
        geocoder = gmaps.GoogleGeocoder(settings.GOOGLE_API_KEY)
        geocoder.bounds = settings.GEO_BOUNDS
        geocoder.region = settings.GEO_REGION
    else:
        geocoder = arcgis.ArcGISCoder(settings.ARCGIS_CLIENT_ID,
                                      settings.ARCGIS_CLIENT_SECRET)

    if page is not None:
        proposals_json = scrape.get_proposals_for_page(page, geocoder)
    else:
        if not since:
            # If there was no last run, the scraper will fetch all
            # proposals.
            since = last_run()

            if not since:
                latest_proposal = Proposal.objects.latest()
                if latest_proposal:
                    since = latest_proposal.updated

            if not since:
                # If there is no record of a previous run, fetch
                # proposals posted since the previous Monday.
                now = datetime.now().replace(hour=0,
                                             minute=0,
                                             second=0,
                                             microsecond=0)
                since = now - timedelta(days=7 + now.weekday())

        proposals_json = scrape.get_proposals_since(dt=since,
                                                    geocoder=geocoder)

    proposals = []

    for p_dict in proposals_json:
        p = create_proposal_from_json(p_dict)

        if p:
            p.save()
            proposals.append(p)
        else:
            logger.error("Could not create proposal from dictionary:", p_dict)

    return proposals
コード例 #3
0
ファイル: tasks.py プロジェクト: johnsliao/cornerwise
def scrape_reports_and_decisions(since=None, page=None,
                                 coder_type=settings.GEOCODER):
    """
    Task that scrapes the reports and decisions page
    """
    if coder_type == "google":
        geocoder = gmaps.GoogleGeocoder(settings.GOOGLE_API_KEY)
        geocoder.bounds = settings.GEO_BOUNDS
        geocoder.region = settings.GEO_REGION
    else:
        geocoder = arcgis.ArcGISCoder(settings.ARCGIS_CLIENT_ID,
                                      settings.ARCGIS_CLIENT_SECRET)

    if page is not None:
        proposals_json = scrape.get_proposals_for_page(page, geocoder)
    else:
        if not since:
            # If there was no last run, the scraper will fetch all
            # proposals.
            since = last_run()

            if not since:
                latest_proposal = Proposal.objects.latest()
                if latest_proposal:
                    since = latest_proposal.updated

            if not since:
                # If there is no record of a previous run, fetch
                # proposals posted since the previous Monday.
                now = datetime.now().replace(hour=0, minute=0,
                                             second=0, microsecond=0)
                since = now - timedelta(days=7 + now.weekday())

        proposals_json = scrape.get_proposals_since(dt=since, geocoder=geocoder)

    proposals = []

    for p_dict in proposals_json:
        p = create_proposal_from_json(p_dict)

        if p:
            p.save()
            proposals.append(p)
        else:
            logger.error("Could not create proposal from dictionary:",
                         p_dict)

    return proposals