def update_stream(): """The mother of all tasks to update the stream with the latest data.""" logging.basicConfig(level=logging.DEBUG, filename="kofkofmtl.log") log = logging.getLogger("Stream Updater") data = fetch_for() now = datetime.datetime.now() found = False for station in data.stations: for measurement in station.measurements: if measurement.hour == now.hour: pollutants = [ k for (k,v) in measurement.pollutants.items() if v > 51 ] if pollutants: found = True log.debug("The measurement at %r at hour %d has the following: %r" % ( station.guess_name(), measurement.hour, measurement.pollutants )) send_air_message(station.guess_name(), pollutants) if not found: log.debug("No records tweeted on %s" % now)
def dbdump_sqlite(dbfile, tablename, days = 5, valuetranslatefunc = None): if not os.path.isdir(os.path.dirname(dbfile)): log.error('DB file %s\' directory does not exist.' %(dbfile)) return False conn = sqlite3.connect(dbfile) log.info('Connected to mysql db at %s' %(dbfile)) c = conn.cursor() query = 'create table if not exists %s(' \ 'id integer primary key, sampledate ts, stationid int, ' \ 'co int, h2s int, no int, no2 int, pm2_5 int, pm2_5f int, pm10 int, o3 int, so2 int );' %(tablename) log.info('Executing sql query: """%s"""' %(query)) c.execute(query) conn.commit() log.info('Starting dump for %d days' %(days)) for n in range(days): today = datetime.datetime.today() - datetime.timedelta(n) try: data = fetcher.fetch_for(today) except Exception, e: log.error('Exception when parsing data for %r: %s' %(today, str(e))) continue for station in data.stations: for measure in station.measurements: datestamp = datetime.datetime(data.year, data.month, data.day, measure.hour) query = 'insert into %s ( sampledate, stationid, ' %(tablename) pols = ', '.join([pollutantToColumn[x.lower()] for x in measure.pollutants.keys()]) query += pols + ') VALUES ( ?, %d, ' %(station.id) query += ', '.join([str(x) for x in measure.pollutants.values()]) query += ");" log.info('Executing query: %s' %(query)) c.execute(query, (datestamp,)) conn.commit()
def grab_data(days=5): """Grabs the data for the last 5 days and outputs to a csv file called data.csv.""" with open("data.csv", "w") as output_file: writer = csv.writer(output_file) for n in range(days): today = datetime.datetime.today() - datetime.timedelta(n) print "Doing %s" % today data = fetch_for(today) writer.writerow(["Year", "Month", "Day"]) writer.writerow([data.year, data.month, data.day]) writer.writerow([]) writer.writerow(["Station Id", "Station Name", "Hour", "Pollutant", "Value"]) for station in data.stations: for measure in station.measurements: for pollutant, value in measure.pollutants.items(): writer.writerow([station.id, station.name, measure.hour, pollutant, value]) writer.writerow([])