def _initialize_store_if_needed(): global gaugedStore global logStore # I think gauged can deal with concurrency concerns since # it's backed by MySQL; in this context it's ok if it fails too if gaugedStore is None: # print 'Initializing gauged store.' gaugedStore = Gauged(configerator.MYSQL_STORE_STRING) gaugedStore.sync() # initiate schema if necessary. if logStore is None: logStore = logging.getLogger(__name__) logStore.setLevel(logging.INFO) handler = logging.FileHandler(BACKUP_LOGFILE) handler.setLevel(logging.INFO) formatter = logging.Formatter('%(message)s') handler.setFormatter(formatter) logStore.addHandler(handler)
benchmark.add_argument( '-d', '--driver', default='sqlite://', help='Where to store the data (defaults to SQLite in-memory)') benchmark.add_argument( '-b', '--block-size', type=int, default=Gauged.DAY, help='The block size to use') benchmark.add_argument( '-r', '--resolution', type=int, default=Gauged.SECOND, help='The resolution to use') options = vars(benchmark.parse_args()) # Setup the Gauged instance gauged = Gauged(options['driver'], block_size=options['block_size'], resolution=options['resolution'], key_overflow=Gauged.IGNORE, gauge_nan=Gauged.IGNORE) gauged.sync() print 'Writing to %s (block_size=%s, resolution=%s)' % \ (options['driver'], options['block_size'], options['resolution']) # Get the start and end timestamp end = datetime.now() start = end - timedelta(days=options['days']) start_timestamp = timegm(start.timetuple()) end_timestamp = timegm(end.timetuple()) number = abbreviate_number(options['number']) print 'Spreading %s measurements to key "foobar" over %s days' % \ (number, options['days'])
help='How many measurements to store') benchmark.add_argument('-t', '--days', type=int, default=365, help='How many days to spread the measurements over') benchmark.add_argument('-d', '--driver', default='sqlite://', help='Where to store the data (defaults to SQLite in-memory)') benchmark.add_argument('-b', '--block-size', type=int, default=Gauged.DAY, help='The block size to use') benchmark.add_argument('-r', '--resolution', type=int, default=Gauged.SECOND, help='The resolution to use') options = vars(benchmark.parse_args()) # Setup the Gauged instance gauged = Gauged(options['driver'], block_size=options['block_size'], resolution=options['resolution'], key_overflow=Gauged.IGNORE, gauge_nan=Gauged.IGNORE) gauged.sync() print 'Writing to %s (block_size=%s, resolution=%s)' % (options['driver'], options['block_size'], options['resolution']) # Get the start and end timestamp end = datetime.now() start = end - timedelta(days=options['days']) start_timestamp = long(timegm(start.timetuple()) * 1000) end_timestamp = long(timegm(end.timetuple()) * 1000) number = abbreviate_number(options['number']) print 'Spreading %s measurements to key "foobar" over %s days' % (number, options['days']) # Benchmark writes
class Reporter(object): """Object used to report both periodic statistics and events.""" player = None start_time = 0 report_file = None event_file = None report = False gauged = None def __init__(self, player): """Initialise files to save reports to.""" if OPTIONS.gauged: self.gauged = Gauged('mysql://root@localhost/gauged') self.gauged.sync() self.player = player file_name = self.player.directory + '/report.csv' self.report_file = open(file_name, 'w') file_name = self.player.directory + '/event.csv' self.event_file = open(file_name, 'w') self.start() def stop(self): """Stop reporting and close file handles.""" self.report = False try: self.report_file.close() except IOError: pass try: self.event_file.close() except IOError: pass def start(self): """Start reporting thread.""" self.report = True self.start_time = time.time() thread = threading.Thread(target=self.reporter, args=()) thread.daemon = True thread.start() def time_elapsed(self): """Calculate the time elapsed since the start of reporting.""" return round(time.time() - self.start_time, 4) def reporter(self): """Periodic reporting of various stats (every second) to file.""" if OPTIONS.gauged: try: mean = self.gauged.aggregate('bandwidth', Gauged.MEAN) count = self.gauged.aggregate('downloads', Gauged.SUM) print '[gauged]', mean, count except: print '[gauged] exception!' if self.report: thread = threading.Timer(interval=float( OPTIONS.reporting_period), function=self.reporter, args=()) thread.daemon = True thread.start() time_elapsed = self.time_elapsed() if OPTIONS.csv: try: self.report_file.flush() except ValueError: pass try: output = (str(time_elapsed) + "," + str(self.player.download_queue.time_buffer) + "," + str(self.player.download_queue.bandwidth) + "," + str(self.player.download_queue.id_) + "," + str(self.player.playback_queue.time_buffer) + "," + str(self.player.playback_queue.time_position) + "," + str(self.player.playback_queue.bandwidth) + "," + str(self.player.playback_queue.id_) + "," + str(self.player.bandwidth) + "\n") except AttributeError: output = str(time_elapsed) + str(', 0, 0, 0, 0, 0, 0, 0\n') try: self.report_file.write(output) except ValueError: pass if OPTIONS.debug: print("[report] " + output), try: self.report_file.flush() except ValueError: pass def event(self, action, description): """Create a thread to handle event.""" thread = threading.Thread(target=self.event_thread, args=(action, description)) thread.daemon = True thread.start() def event_thread(self, action, description): """Event reporting to file.""" time_elapsed = self.time_elapsed() if OPTIONS.csv: try: self.event_file.flush() except ValueError: pass output = (str(time_elapsed) + "," + str(action) + "," + str(description) + "\n") try: self.event_file.write(output) except ValueError: pass if OPTIONS.debug: print("[event] " + output), try: self.event_file.flush() except ValueError: pass def gauged_event(self, **gauged_data): """ Create a thread to handle event.""" if OPTIONS.gauged: thread = threading.Thread(target=self.gauged_event_thread, kwargs=gauged_data) thread.daemon = True thread.start() def gauged_event_thread(self, **gauged_data): """Event reporting to gauged.""" try: with self.gauged.writer as writer: writer.add(gauged_data) except: pass
class Reporter(object): """Object used to report both periodic statistics and events.""" player = None start_time = 0 report_file = None event_file = None report = False gauged = None def __init__(self, player): """Initialise files to save reports to.""" if OPTIONS.gauged: self.gauged = Gauged('mysql://root@localhost/gauged') self.gauged.sync() self.player = player file_name = self.player.directory + '/report.csv' self.report_file = open(file_name, 'w') file_name = self.player.directory + '/event.csv' self.event_file = open(file_name, 'w') self.start() def stop(self): """Stop reporting and close file handles.""" self.report = False try: self.report_file.close() except IOError: pass try: self.event_file.close() except IOError: pass def start(self): """Start reporting thread.""" self.report = True self.start_time = time.time() thread = threading.Thread(target=self.reporter, args=()) thread.daemon = True thread.start() def time_elapsed(self): """Calculate the time elapsed since the start of reporting.""" return round(time.time() - self.start_time, 4) def reporter(self): """Periodic reporting of various stats (every second) to file.""" if OPTIONS.gauged: try: mean = self.gauged.aggregate('bandwidth', Gauged.MEAN) count = self.gauged.aggregate('downloads', Gauged.SUM) print '[gauged]', mean, count except: print '[gauged] exception!' if self.report: thread = threading.Timer( interval=float(OPTIONS.reporting_period), function=self.reporter, args=()) thread.daemon = True thread.start() time_elapsed = self.time_elapsed() if OPTIONS.csv: try: self.report_file.flush() except ValueError: pass try: output = (str(time_elapsed) + "," + str(self.player.download_queue.time_buffer) + "," + str(self.player.download_queue.bandwidth) + "," + str(self.player.download_queue.id_) + "," + str(self.player.playback_queue.time_buffer) + "," + str(self.player.playback_queue.time_position) + "," + str(self.player.playback_queue.bandwidth) + "," + str(self.player.playback_queue.id_) + "," + str(self.player.bandwidth) + "\n") except AttributeError: output = str(time_elapsed) + str(', 0, 0, 0, 0, 0, 0, 0\n') try: self.report_file.write(output) except ValueError: pass if OPTIONS.debug: print ("[report] " + output), try: self.report_file.flush() except ValueError: pass def event(self, action, description): """Create a thread to handle event.""" thread = threading.Thread(target=self.event_thread, args=(action, description)) thread.daemon = True thread.start() def event_thread(self, action, description): """Event reporting to file.""" time_elapsed = self.time_elapsed() if OPTIONS.csv: try: self.event_file.flush() except ValueError: pass output = (str(time_elapsed) + "," + str(action) + "," + str(description) + "\n") try: self.event_file.write(output) except ValueError: pass if OPTIONS.debug: print ("[event] " + output), try: self.event_file.flush() except ValueError: pass def gauged_event(self, **gauged_data): """ Create a thread to handle event.""" if OPTIONS.gauged: thread = threading.Thread(target=self.gauged_event_thread, kwargs=gauged_data) thread.daemon = True thread.start() def gauged_event_thread(self, **gauged_data): """Event reporting to gauged.""" try: with self.gauged.writer as writer: writer.add(gauged_data) except: pass