def update_stream():
    """The mother of all tasks to update the stream with the latest data."""
    logging.basicConfig(level=logging.DEBUG, filename="kofkofmtl.log")
    log = logging.getLogger("Stream Updater")
    data = fetch_for()
    now = datetime.datetime.now()
    found = False
    for station in data.stations:
        for measurement in station.measurements:
            if measurement.hour == now.hour:
                pollutants = [
                    k for (k,v) in measurement.pollutants.items()
                        if v > 51
                ]
                if pollutants:
                    found = True
                    log.debug("The measurement at %r at hour %d has the following: %r" % (
                        station.guess_name(),
                        measurement.hour,
                        measurement.pollutants
                    ))
                    send_air_message(station.guess_name(), pollutants)
    
    if not found:
        log.debug("No records tweeted on %s" % now)
def dbdump_sqlite(dbfile, tablename, days = 5, valuetranslatefunc = None):
    if not os.path.isdir(os.path.dirname(dbfile)):
        log.error('DB file %s\' directory does not exist.' %(dbfile))
        return False
    
    conn = sqlite3.connect(dbfile)
    log.info('Connected to mysql db at %s' %(dbfile))
    c = conn.cursor()
    query = 'create table if not exists %s(' \
        'id integer primary key, sampledate ts, stationid int, ' \
        'co int, h2s int, no int, no2 int, pm2_5 int, pm2_5f int, pm10 int, o3 int, so2 int );' %(tablename)              
    log.info('Executing sql query: """%s"""' %(query))
    c.execute(query)
    conn.commit()

    log.info('Starting dump for %d days' %(days))
    for n in range(days):
        today = datetime.datetime.today() - datetime.timedelta(n)
        try:
            data = fetcher.fetch_for(today)
        except Exception, e:
            log.error('Exception when parsing data for %r: %s' %(today, str(e)))
            continue
        
        for station in data.stations:
            for measure in station.measurements:
                datestamp = datetime.datetime(data.year, data.month, data.day, measure.hour)
                query = 'insert into %s ( sampledate, stationid, ' %(tablename)
                pols = ', '.join([pollutantToColumn[x.lower()] for x in measure.pollutants.keys()])
                query += pols + ') VALUES ( ?, %d, ' %(station.id)
                query += ', '.join([str(x) for x in measure.pollutants.values()])
                query += ");"
                log.info('Executing query: %s' %(query))
                c.execute(query, (datestamp,))
        conn.commit()
def grab_data(days=5):
    """Grabs the data for the last 5 days and outputs to a csv file called data.csv."""

    with open("data.csv", "w") as output_file:
        writer = csv.writer(output_file)
        for n in range(days):
            today = datetime.datetime.today() - datetime.timedelta(n)
            print "Doing %s" % today
            data = fetch_for(today)

            writer.writerow(["Year", "Month", "Day"])
            writer.writerow([data.year, data.month, data.day])
            writer.writerow([])

            writer.writerow(["Station Id", "Station Name", "Hour", "Pollutant", "Value"])
            for station in data.stations:
                for measure in station.measurements:
                    for pollutant, value in measure.pollutants.items():
                        writer.writerow([station.id, station.name, measure.hour, pollutant, value])
            writer.writerow([])