def update_stream():
"""The mother of all tasks to update the stream with the latest data."""
logging.basicConfig(level=logging.DEBUG, filename="kofkofmtl.log")
log = logging.getLogger("Stream Updater")
data = fetch_for()
now = datetime.datetime.now()
found = False
for station in data.stations:
for measurement in station.measurements:
if measurement.hour == now.hour:
pollutants = [
k for (k,v) in measurement.pollutants.items()
if v > 51
]
if pollutants:
found = True
log.debug("The measurement at %r at hour %d has the following: %r" % (
station.guess_name(),
measurement.hour,
measurement.pollutants
))
send_air_message(station.guess_name(), pollutants)
if not found:
log.debug("No records tweeted on %s" % now)
def dbdump_sqlite(dbfile, tablename, days = 5, valuetranslatefunc = None):
if not os.path.isdir(os.path.dirname(dbfile)):
log.error('DB file %s\' directory does not exist.' %(dbfile))
return False
conn = sqlite3.connect(dbfile)
log.info('Connected to mysql db at %s' %(dbfile))
c = conn.cursor()
query = 'create table if not exists %s(' \
'id integer primary key, sampledate ts, stationid int, ' \
'co int, h2s int, no int, no2 int, pm2_5 int, pm2_5f int, pm10 int, o3 int, so2 int );' %(tablename)
log.info('Executing sql query: """%s"""' %(query))
c.execute(query)
conn.commit()
log.info('Starting dump for %d days' %(days))
for n in range(days):
today = datetime.datetime.today() - datetime.timedelta(n)
try:
data = fetcher.fetch_for(today)
except Exception, e:
log.error('Exception when parsing data for %r: %s' %(today, str(e)))
continue
for station in data.stations:
for measure in station.measurements:
datestamp = datetime.datetime(data.year, data.month, data.day, measure.hour)
query = 'insert into %s ( sampledate, stationid, ' %(tablename)
pols = ', '.join([pollutantToColumn[x.lower()] for x in measure.pollutants.keys()])
query += pols + ') VALUES ( ?, %d, ' %(station.id)
query += ', '.join([str(x) for x in measure.pollutants.values()])
query += ");"
log.info('Executing query: %s' %(query))
c.execute(query, (datestamp,))
conn.commit()
def grab_data(days=5):
"""Grabs the data for the last 5 days and outputs to a csv file called data.csv."""
with open("data.csv", "w") as output_file:
writer = csv.writer(output_file)
for n in range(days):
today = datetime.datetime.today() - datetime.timedelta(n)
print "Doing %s" % today
data = fetch_for(today)
writer.writerow(["Year", "Month", "Day"])
writer.writerow([data.year, data.month, data.day])
writer.writerow([])
writer.writerow(["Station Id", "Station Name", "Hour", "Pollutant", "Value"])
for station in data.stations:
for measure in station.measurements:
for pollutant, value in measure.pollutants.items():
writer.writerow([station.id, station.name, measure.hour, pollutant, value])
writer.writerow([])