def get_all_events(cnx, date_range, next_range): # Get all the events (including EOD .exists) for a day query = ("SELECT stacktach_rawdata.when AS d, " "stacktach_rawdata.event as event, " "stacktach_rawdata.json AS rawjson " "FROM stacktach_rawdata " "WHERE (stacktach_rawdata.when BETWEEN %f AND %f) " "ORDER BY stacktach_rawdata.when" % date_range) cursor = cnx.cursor() cursor.execute(query) payloads = [] for when, event, rawjson in cursor: hwhen = nu.dt_from_decimal(when) queue, full = json.loads(rawjson) payload = full['payload'] if event == 'compute.instance.exists': astart = dateutil.parser.parse(payload['audit_period_beginning']) aend = dateutil.parser.parse(payload['audit_period_ending']) # Ignore these EOD events, they're in tomorrow's batch. if (astart.time() == datetime.time.min and aend.time() == datetime.time.min): # print "EOD .exists at: %s (%s -> %s)" % (hwhen, astart, aend) continue payloads.append((hwhen, full)) cursor.close() query = ("SELECT stacktach_rawdata.when AS d, " "stacktach_rawdata.json AS rawjson " "FROM stacktach_rawdata " "WHERE (stacktach_rawdata.when BETWEEN %f AND %f) " "AND stacktach_rawdata.event='compute.instance.exists' " "ORDER BY stacktach_rawdata.when" % next_range) cursor = cnx.cursor() cursor.execute(query) for when, rawjson in cursor: hwhen = nu.dt_from_decimal(when) queue, full = json.loads(rawjson) payload = full['payload'] astart = dateutil.parser.parse(payload['audit_period_beginning']) aend = dateutil.parser.parse(payload['audit_period_ending']) # Ignore these EOD events, they're in tomorrow's batch. if (astart.time() != datetime.time.min or aend.time() != datetime.time.min): # print "Instant .exists at: %s (%s -> %s)" % (hwhen, astart, aend) continue payloads.append((hwhen, full)) cursor.close() print "%d events for time period" % (len(payloads), ) return payloads
channel=connection.channel()) queue.declare() cnx = mysql.connector.connect(user='******', password='******', host='127.0.0.1', database='stacktach') cursor = cnx.cursor() query = ("SELECT min(stacktach_rawdata.when) AS mindate, " "MAX(stacktach_rawdata.when) AS maxdate " "FROM stacktach_rawdata") cursor.execute(query) mindate, maxdate = list(cursor)[0] cursor.close() oldest = nu.dt_from_decimal(mindate) newest = nu.dt_from_decimal(maxdate) # Move to the next day so we get all the events for the day. start = oldest.replace(hour=0,minute=0,second=0,microsecond=0) \ + datetime.timedelta(days=1) end = start + datetime.timedelta(days=1, hours=4) print "Events from %s to %s" % (oldest, newest) print "Extracting events from %s to %s" % (start, end) time.sleep(5) dstart = nu.dt_to_decimal(start) dend = nu.dt_to_decimal(end)