def setUp(self):
self.test_data_dir = os.path.join(os.path.dirname(__file__), "testLivesData")
try:
print "cleaning out export dir"
shutil.rmtree(self.test_data_dir)
except OSError:
print "nothing to clean out"
self.collection = mongolab.connect().va
self.lives = LivesDataExporter(self.collection, 'Norfolk', self.test_data_dir)
def setUp(self):
self.test_data_dir = os.path.join(os.path.dirname(__file__),
"testLivesData")
try:
print "cleaning out export dir"
shutil.rmtree(self.test_data_dir)
except OSError:
print "nothing to clean out"
self.collection = mongolab.connect().va
self.lives = LivesDataExporter(self.collection, 'Norfolk',
self.test_data_dir)
def scheduled_job():
c = config.load()
print 'Connect to DB'
db = mongolab.connect()
dest_collection = db[c['state_abb']]
# collection of establishments to fetch
to_fetch = db['establishments_to_fetch']
if to_fetch.find_one() is None:
print 'Make a list of Establishments to Fetch'
print 'Finding Cities'
cities = scraper.get_cities()
for city in cities:
print 'Finding Establishments in ' + city['name']
establishments = scraper.get_establishments(city)
print 'Found ' + str(len(establishments)) + ' in ' + city['name']
for establishment in establishments:
to_fetch.insert(establishment)
print 'Start Fetching Establishment Data'
establishments = to_fetch.find()
added = updated = 0
for establishment in establishments:
fetch_id = establishment['_id']
existing = dest_collection.find_one({'url': establishment['url']})
if existing is not None:
print existing
if 'last_inspected_date' not in existing or \
existing['last_inspected_date'] < establishment['last_inspected_date']:
changed_fields = []
establishment = scraper.get_establishment_details(establishment)
establishment['inspections'] = scraper.get_inspections(establishment, establishment['baseUrl'])
del establishment['baseUrl'] # Need to remove 'baseUrl' and 'inserted' before comparing or inserting
del establishment['inserted']
establishment['_id'] = existing['_id'] # Must set establishment id to existing or it won't update
# correctly
changed_fields = list(o for o in establishment if existing[o] != establishment[o])
else:
continue
else:
establishment = scraper.get_establishment_details(establishment)
establishment['inspections'] = scraper.get_inspections(establishment, establishment['baseUrl'])
establishment['_id'] = None # This is necessary to get a new ID from mongoDB when inserting
changed_fields = ['None']
if changed_fields:
establishment = scraper.get_establishment_geo(establishment) # This is required to update coordinates
dest_collection.update({'_id': establishment['_id']},
establishment,
True)
if 'None' in changed_fields:
print '\t' + establishment['name'] + ' Added!'
added += 1
else:
print '\t' + establishment['name'] + ' Updated!'
updated += 1
to_fetch.remove({'_id': fetch_id})
print str(added) + ' new establishments added'
print str(updated) + ' existing establishments updated'
import config
import mongolab
from pymongo import errors
c = config.load()
db = mongolab.connect()
va_establishments = db[c['state_abb']]
categories = {'Seasonal Fast Food Restaurant': 'Restaurant',
'Fast Food Restaurant': 'Restaurant',
'Full Service Restaurant': 'Restaurant',
'Public Middle or High School Food Service': 'Education',
'Mobile Food Unit': 'Mobile Food',
'Private Elementary School Food Service': 'Education',
'Child Care Food Service': 'Education',
'Other Food Service': 'Other',
'Mobile food unit': 'Mobile Food',
'Public Elementary School Food Service': 'Education',
'Dept. of Juvenile Justice Food Service': 'Government',
'Carry Out Food Service Only': 'Grocery',
'Commissary': 'Grocery',
'Hotel Continental Breakfast': 'Hospitality',
'Full Service Restaurant/Caterer': 'Restaurant',
'Hospital Food Service': 'Medical',
'Caterer': 'Restaurant',
'State College Food Service': 'Education',
'Convenience Store Food Service': 'Grocery',
'Private Middle or High School Food Service': 'Education',
'Bed & Breakfast Food Service': 'Hospitality',
'Adult Care Home Food Service': 'Medical',
def get_db():
if not hasattr(g, 'db'):
g.db = mongolab.connect()
import os
import re
from math import radians, cos, sin, atan2, sqrt
from collections import OrderedDict
from datetime import datetime
from flask import Flask, Response, url_for, request, current_app, render_template
from functools import wraps
from livesdataexporter import LivesDataExporter
from piwik import Tracker
from threading import Thread
app = Flask(__name__)
app.debug = False
try:
db = mongolab.connect()
except ValueError:
print "Could not connect to database"
piwik = Tracker()
@app.before_request
def before_request():
if piwik:
piwik.track(request)
def support_jsonp(f):
"""Wraps JSONified output for JSONP"""
@wraps(f)
def get_db():
if not hasattr(g, 'db'):
g.db = mongolab.connect()
