def setUp(self): self.test_data_dir = os.path.join(os.path.dirname(__file__), "testLivesData") try: print "cleaning out export dir" shutil.rmtree(self.test_data_dir) except OSError: print "nothing to clean out" self.collection = mongolab.connect().va self.lives = LivesDataExporter(self.collection, 'Norfolk', self.test_data_dir)
def scheduled_job(): c = config.load() print 'Connect to DB' db = mongolab.connect() dest_collection = db[c['state_abb']] # collection of establishments to fetch to_fetch = db['establishments_to_fetch'] if to_fetch.find_one() is None: print 'Make a list of Establishments to Fetch' print 'Finding Cities' cities = scraper.get_cities() for city in cities: print 'Finding Establishments in ' + city['name'] establishments = scraper.get_establishments(city) print 'Found ' + str(len(establishments)) + ' in ' + city['name'] for establishment in establishments: to_fetch.insert(establishment) print 'Start Fetching Establishment Data' establishments = to_fetch.find() added = updated = 0 for establishment in establishments: fetch_id = establishment['_id'] existing = dest_collection.find_one({'url': establishment['url']}) if existing is not None: print existing if 'last_inspected_date' not in existing or \ existing['last_inspected_date'] < establishment['last_inspected_date']: changed_fields = [] establishment = scraper.get_establishment_details(establishment) establishment['inspections'] = scraper.get_inspections(establishment, establishment['baseUrl']) del establishment['baseUrl'] # Need to remove 'baseUrl' and 'inserted' before comparing or inserting del establishment['inserted'] establishment['_id'] = existing['_id'] # Must set establishment id to existing or it won't update # correctly changed_fields = list(o for o in establishment if existing[o] != establishment[o]) else: continue else: establishment = scraper.get_establishment_details(establishment) establishment['inspections'] = scraper.get_inspections(establishment, establishment['baseUrl']) establishment['_id'] = None # This is necessary to get a new ID from mongoDB when inserting changed_fields = ['None'] if changed_fields: establishment = scraper.get_establishment_geo(establishment) # This is required to update coordinates dest_collection.update({'_id': establishment['_id']}, establishment, True) if 'None' in changed_fields: print '\t' + establishment['name'] + ' Added!' added += 1 else: print '\t' + establishment['name'] + ' Updated!' updated += 1 to_fetch.remove({'_id': fetch_id}) print str(added) + ' new establishments added' print str(updated) + ' existing establishments updated'
import config import mongolab from pymongo import errors c = config.load() db = mongolab.connect() va_establishments = db[c['state_abb']] categories = {'Seasonal Fast Food Restaurant': 'Restaurant', 'Fast Food Restaurant': 'Restaurant', 'Full Service Restaurant': 'Restaurant', 'Public Middle or High School Food Service': 'Education', 'Mobile Food Unit': 'Mobile Food', 'Private Elementary School Food Service': 'Education', 'Child Care Food Service': 'Education', 'Other Food Service': 'Other', 'Mobile food unit': 'Mobile Food', 'Public Elementary School Food Service': 'Education', 'Dept. of Juvenile Justice Food Service': 'Government', 'Carry Out Food Service Only': 'Grocery', 'Commissary': 'Grocery', 'Hotel Continental Breakfast': 'Hospitality', 'Full Service Restaurant/Caterer': 'Restaurant', 'Hospital Food Service': 'Medical', 'Caterer': 'Restaurant', 'State College Food Service': 'Education', 'Convenience Store Food Service': 'Grocery', 'Private Middle or High School Food Service': 'Education', 'Bed & Breakfast Food Service': 'Hospitality', 'Adult Care Home Food Service': 'Medical',
def get_db(): if not hasattr(g, 'db'): g.db = mongolab.connect()
import os import re from math import radians, cos, sin, atan2, sqrt from collections import OrderedDict from datetime import datetime from flask import Flask, Response, url_for, request, current_app, render_template from functools import wraps from livesdataexporter import LivesDataExporter from piwik import Tracker from threading import Thread app = Flask(__name__) app.debug = False try: db = mongolab.connect() except ValueError: print "Could not connect to database" piwik = Tracker() @app.before_request def before_request(): if piwik: piwik.track(request) def support_jsonp(f): """Wraps JSONified output for JSONP""" @wraps(f)