def _evaluate(experiment: EvExperiment, dao: Dao, statsd: StatsClient): try: with statsd.timer('timing.evaluation'): _logger.info(f'Evaluating experiment [{experiment.id}]') _logger.debug( f'Loading goals for experiment [{experiment.id}]') with statsd.timer('timing.query'): goals = dao.get_agg_goals(experiment).sort_values( ['exp_variant_id', 'goal']) _logger.info( f'Retrieved {len(goals)} goals in experiment [{experiment.id}]' ) with statsd.timer('timing.stats'): evaluation = experiment.evaluate_agg(goals) statsd.incr('evaluations') _logger.info(( f'Evaluation of experiment [{experiment.id}] finished with evaluation' f' of {evaluation.metrics.metric_id.nunique()} ' f'metrics and {evaluation.checks.check_id.nunique()} checks' )) return Result.from_evaluation(experiment, evaluation) except Exception as e: _logger.error( f'Cannot evaluate experiment [{experiment.id}] because of {e}') _logger.exception(e) statsd.incr('errors.experiment') raise HTTPException( status_code=500, detail= f'Cannot evaluate experiment [{experiment.id}] because of {e}', )
class StatsdMetrics(Metrics): def __init__(self, host='localhost', port=8125, prefix=None): self.statsd = StatsClient(host, port, prefix) def fanout_timer(self, feed_class): return self.statsd.timer('%s.fanout_latency' % feed_class.__name__) def feed_reads_timer(self, feed_class): return self.statsd.timer('%s.read_latency' % feed_class.__name__) def on_feed_read(self, feed_class, activities_count): self.statsd.incr('%s.reads' % feed_class.__name__, activities_count) def on_feed_write(self, feed_class, activities_count): self.statsd.incr('%s.writes' % feed_class.__name__, activities_count) def on_feed_remove(self, feed_class, activities_count): self.statsd.incr('%s.deletes' % feed_class.__name__, activities_count) def on_fanout(self, feed_class, operation, activities_count=1): metric = (feed_class.__name__, operation.__name__) self.statsd.incr('%s.fanout.%s' % metric, activities_count) def on_activity_published(self): self.statsd.incr('activities.published') def on_activity_removed(self): self.statsd.incr('activities.removed')
class StatsdWrapper: """Simple wrapper around the statsd client.""" statsd = None def __init__(self, host, port, prefix): if host: self.statsd = StatsClient( host=host, port=port, prefix=prefix, ) def incr(self, *args): if self.statsd: self.statsd.incr(*args) def decr(self, *args): if self.statsd: self.statsd.decr(*args) def gauge(self, *args): if self.statsd: self.statsd.gauge(*args) def timing(self, *args): if self.statsd: self.statsd.timing(*args) def timer(self, *args): if self.statsd: self.statsd.timer(*args) def set(self, *args): if self.statsd: self.statsd.set(*args)
class FlaskStat(object): _xstat_title = None _xstat_host = None _xstat_port = None _stat_client = None def __init__(self, app=None): super(FlaskStat, self).__init__() if app: self.init_app(app) def init_app(self, app): from flask import request, g """ 绑定app """ self._xstat_title = app.config.get('XSTAT_TITLE') self._xstat_host = app.config.get('XSTAT_HOST') self._xstat_port = app.config.get('XSTAT_PORT') or constants.XSTAT_PORT self._stat_client = StatsClient(host=self._xstat_host, port=self._xstat_port) @app.before_request @catch_exc def prepare_stat(): if not request.endpoint: return g.xstat_timers = [] g.xstat_timers.append( self._stat_client.timer('.'.join([ self._xstat_title, 'endpoint', request.endpoint, ]) ) ) g.xstat_timers.append( self._stat_client.timer('.'.join([ self._xstat_title, 'all', ]) ) ) for stat in g.xstat_timers: stat.start() @app.teardown_request @catch_exc def send_stat(exc): if not hasattr(g, 'xstat_timers'): return for stat in g.xstat_timers: stat.stop()
class MapleStat(object): _xstat_title = None _xstat_host = None _xstat_port = None _stat_client = None def __init__(self, app=None, config=None): super(MapleStat, self).__init__() if app: self.init_app(app, config) def init_app(self, app, config): """ 绑定app """ self._xstat_title = config.get('XSTAT_TITLE') self._xstat_host = config.get('XSTAT_HOST') self._xstat_port = config.get('XSTAT_PORT') or constants.XSTAT_PORT self._stat_client = StatsClient(host=self._xstat_host, port=self._xstat_port) @app.before_request @catch_exc def prepare_stat(request): if not request.endpoint: return request.xstat_timers = [] request.xstat_timers.append( self._stat_client.timer('.'.join([ self._xstat_title, 'endpoint', request.endpoint, ]) ) ) request.xstat_timers.append( self._stat_client.timer('.'.join([ self._xstat_title, 'all', ]) ) ) for stat in request.xstat_timers: stat.start() @app.after_request @catch_exc def send_stat(request, exc): if not hasattr(request, 'xstat_timers'): return for stat in request.xstat_timers: stat.stop()
class DjangoStat(MiddlewareMixin): _xstat_title = None _xstat_host = None _xstat_port = None _stat_client = None def __init__(self, *args, **kwargs): from django.conf import settings super(DjangoStat, self).__init__(*args, **kwargs) self._xstat_title = getattr(settings, 'XSTAT_TITLE', None) self._xstat_host = getattr(settings, 'XSTAT_HOST', None) self._xstat_port = getattr(settings, 'XSTAT_PORT', None) or constants.XSTAT_PORT self._stat_client = StatsClient(host=self._xstat_host, port=self._xstat_port) @catch_exc def process_view(self, request, view_func, view_args, view_kwargs): """ request.resolver_match.url_name 在process_view才可以取到 :return: """ request.xstat_timers = [] request.xstat_timers.append( self._stat_client.timer('.'.join([ self._xstat_title, 'endpoint', request.resolver_match.url_name, ]) ) ) request.xstat_timers.append( self._stat_client.timer('.'.join([ self._xstat_title, 'all', ]) ) ) for stat in request.xstat_timers: stat.start() @catch_exc def process_response(self, request, response): """ 无论是否抛出异常,都会执行这一步 """ if not hasattr(request, 'xstat_timers'): return response for stat in request.xstat_timers: stat.stop() return response
class FlaskStat(object): _xstat_title = None _xstat_host = None _xstat_port = None _stat_client = None def __init__(self, app=None): super(FlaskStat, self).__init__() if app: self.init_app(app) def init_app(self, app): from flask import request, g """ 绑定app """ self._xstat_title = app.config.get('XSTAT_TITLE') self._xstat_host = app.config.get('XSTAT_HOST') self._xstat_port = app.config.get('XSTAT_PORT') or constants.XSTAT_PORT self._stat_client = StatsClient(host=self._xstat_host, port=self._xstat_port) @app.before_request @catch_exc def prepare_stat(): if not request.endpoint: return g.xstat_timers = [] g.xstat_timers.append( self._stat_client.timer('.'.join([ self._xstat_title, 'endpoint', request.endpoint, ]))) g.xstat_timers.append( self._stat_client.timer('.'.join([ self._xstat_title, 'all', ]))) for stat in g.xstat_timers: stat.start() @app.teardown_request @catch_exc def send_stat(exc): if not hasattr(g, 'xstat_timers'): return for stat in g.xstat_timers: stat.stop()
class DjangoStat(object): _xstat_title = None _xstat_host = None _xstat_port = None _stat_client = None def __init__(self): from django.conf import settings super(DjangoStat, self).__init__() self._xstat_title = getattr(settings, 'XSTAT_TITLE', None) self._xstat_host = getattr(settings, 'XSTAT_HOST', None) self._xstat_port = getattr(settings, 'XSTAT_PORT', None) or constants.XSTAT_PORT self._stat_client = StatsClient(host=self._xstat_host, port=self._xstat_port) @catch_exc def process_view(self, request, view_func, view_args, view_kwargs): """ request.resolver_match.url_name 在process_view才可以取到 :return: """ request.xstat_timers = [] request.xstat_timers.append( self._stat_client.timer('.'.join([ self._xstat_title, 'endpoint', request.resolver_match.url_name, ]))) request.xstat_timers.append( self._stat_client.timer('.'.join([ self._xstat_title, 'all', ]))) for stat in request.xstat_timers: stat.start() @catch_exc def process_response(self, request, response): """ 无论是否抛出异常,都会执行这一步 """ if not hasattr(request, 'xstat_timers'): return response for stat in request.xstat_timers: stat.stop() return response
class MapleStat(object): _xstat_title = None _xstat_host = None _xstat_port = None _stat_client = None def __init__(self, app=None, config=None): super(MapleStat, self).__init__() if app: self.init_app(app, config) def init_app(self, app, config): """ 绑定app """ self._xstat_title = config.get('XSTAT_TITLE') self._xstat_host = config.get('XSTAT_HOST') self._xstat_port = config.get('XSTAT_PORT') or constants.XSTAT_PORT self._stat_client = StatsClient(host=self._xstat_host, port=self._xstat_port) @app.before_request @catch_exc def prepare_stat(request): if not request.endpoint: return request.xstat_timers = [] request.xstat_timers.append( self._stat_client.timer('.'.join([ self._xstat_title, 'endpoint', request.endpoint, ]))) request.xstat_timers.append( self._stat_client.timer('.'.join([ self._xstat_title, 'all', ]))) for stat in request.xstat_timers: stat.start() @app.after_request @catch_exc def send_stat(request, exc): if not hasattr(request, 'xstat_timers'): return for stat in request.xstat_timers: stat.stop()
class _Statsd(object): def __init__(self, config): if config.get('datadog', True): initialize(statsd_host=config['host'], statsd_port=config['port'], prefix=config['prefix']) self.datadog = True self._statsd = statsd else: self.datadog = False self._statsd = StatsClient(config['host'], config['port'], config['prefix']) def incr(self, metric, count=1, rate=1, **kw): if self.datadog: return self._statsd.increment(metric, value=count, sample_rate=rate, **kw) else: return self._statsd.incr(metric, count=count, rate=rate) def timer(self, metric, rate=1, **kw): if self.datadog: return self._statsd.timed(metric, sample_rate=rate, **kw) else: return self._statsd.timer(metric, rate=rate)
def time_stack_list(username, password, tenant, auth_url, heat_url, region, statsd_server): keystone = keystone_client(username=username, password=password, tenant_name=tenant, auth_url=auth_url) token = keystone.auth_token heat = heat_client('1', endpoint=heat_url, region_name=region, token=token) statsd = StatsClient(host=statsd_server) with statsd.timer('uptime.{}'.format(region)): list(heat.stacks.list())
class StatsD(object): def __init__(self, app=None, config=None): self.config = None self.statsd = None if app is not None: self.init_app(app) else: self.app = None def init_app(self, app, config=None): if config is not None: self.config = config elif self.config is None: self.config = app.config self.config.setdefault('STATSD_HOST', 'localhost') self.config.setdefault('STATSD_PORT', 8125) self.config.setdefault('STATSD_PREFIX', None) self.app = app self.statsd = StatsClient(self.config['STATSD_HOST'], self.config['STATSD_PORT'], self.config['STATSD_PREFIX']) def timer(self, *args, **kwargs): return self.statsd.timer(*args, **kwargs) def timing(self, *args, **kwargs): return self.statsd.timing(*args, **kwargs) def incr(self, *args, **kwargs): return self.statsd.incr(*args, **kwargs) def decr(self, *args, **kwargs): return self.statsd.decr(*args, **kwargs) def gauge(self, *args, **kwargs): return self.statsd.gauge(*args, **kwargs) def set(self, *args, **kwargs): return self.statsd.set(*args, **kwargs)
class StatsD(object): def __init__(self, app=None, config=None): self.config = None self.statsd = None if app is not None: self.init_app(app) else: self.app = None def init_app(self, app, config=None): if config is not None: self.config = config elif self.config is None: self.config = app.config self.config.setdefault("STATSD_HOST", "localhost") self.config.setdefault("STATSD_PORT", 8125) self.config.setdefault("STATSD_PREFIX", None) self.app = app self.statsd = StatsClient( host=self.config["STATSD_HOST"], port=self.config["STATSD_PORT"], prefix=self.config["STATSD_PREFIX"] ) def timer(self, *args, **kwargs): return self.statsd.timer(*args, **kwargs) def timing(self, *args, **kwargs): return self.statsd.timing(*args, **kwargs) def incr(self, *args, **kwargs): return self.statsd.incr(*args, **kwargs) def decr(self, *args, **kwargs): return self.statsd.decr(*args, **kwargs) def gauge(self, *args, **kwargs): return self.statsd.gauge(*args, **kwargs)
class Client(object): def __init__(self, server, zero_fill=True, **kw): self.server = server.rstrip('/') self.session = requests.session() # getting monolith info info = self.session.get(server).json if callable(info): info = info() self.es = self.server + info['es_endpoint'] self.fields = info['fields'] self.zero_fill = zero_fill # statsd settings statsd_host = kw.get('statsd.host', 'localhost') statsd_port = int(kw.get('statsd.port', 8125)) statsd_prefix = kw.get('statsd.prefix', 'monolith.client') self.statsd = StatsClient(host=statsd_host, port=statsd_port, prefix=statsd_prefix) def __call__(self, field, start, end, interval=DAY, strict_range=False, **terms): if isinstance(interval, basestring): interval = _str2interval[interval.encode()] if isinstance(start, basestring): start = datetime.datetime.strptime(start.encode(), '%Y-%m-%d').toordinal() start = datetime.date.fromordinal(start) end = datetime.datetime.strptime(end.encode(), '%Y-%m-%d').toordinal() end = datetime.date.fromordinal(end) if interval == DAY: drange = util.iterdays(start, end) elif interval == WEEK: drange = util.iterweeks(start, end) elif interval == MONTH: drange = util.itermonths(start, end) else: drange = util.iteryears(start, end) # building the query start_date_str = start.strftime('%Y-%m-%d') end_date_str = end.strftime('%Y-%m-%d') if isinstance(interval, int): interval = _interval2str[interval] # XXX we'll see later if we want to provide a # nicer query interface # we need a facet query if strict_range: greater = "gt" lower = "lt" else: greater = "gte" lower = "lte" query = { "query": { "match_all": {}, }, "size": 0, # we aren't interested in the hits "facets": { "histo1": { "date_histogram": { "value_field": field, "interval": interval, "key_field": "date", }, "facet_filter": { "range": { "date": { greater: start_date_str, lower: end_date_str, } } } } } } if len(terms) > 0: term = {} for key, value in terms.items(): term[key] = value range_ = query['facets']['histo1']['facet_filter']['range'] filter_ = {'and': [{'term': term}, {'range': range_}]} query['facets']['histo1']['facet_filter'] = filter_ with self.statsd.timer('elasticsearch-query'): res = self.session.post(self.es, data=json.dumps(query)) if res.status_code != 200: raise ValueError(res.content) # getting the JSON content res = res.json if callable(res): res = res() # statsd calls self.statsd.incr('elasticsearch-call') if not isinstance(res, dict): raise ValueError(res) if 'errors' in res: raise ValueError(res['errors'][0]['description']) dates = set() for entry in res['facets']['histo1']['entries']: time_ = entry['time'] / 1000.0 date_ = datetime.datetime.utcfromtimestamp(time_).date() if 'total' in entry: count = entry['total'] else: count = entry['count'] if date_ not in dates: dates.add(date_) yield {'count': count, 'date': date_} if self.zero_fill: # yielding zeros for date_ in drange: if strict_range and date_ in (start, end): continue if date_ not in dates: yield {'count': 0, 'date': date_}
class StatsD(object): def __init__(self, app=None, config=None): self.config = None self.statsd = None if app is not None: self.init_app(app, config=config) else: self.app = None def init_app(self, app, config=None): if config is not None: self.config = config elif self.config is None: self.config = app.config self.config.setdefault('STATSD_HOST', 'localhost') self.config.setdefault('STATSD_PORT', 8125) self.config.setdefault('STATSD_PREFIX', None) self.app = app self.statsd = StatsClient(self.config['STATSD_HOST'], self.config['STATSD_PORT'], self.config['STATSD_PREFIX']) self.use_ms=self.config.get('STATSD_USEMS', True) # Configure any of our middleware self.setup_middleware() def timer(self, *args, **kwargs): return self.statsd.timer(*args, **kwargs) def timing(self, *args, **kwargs): return self.statsd.timing(*args, **kwargs) def incr(self, *args, **kwargs): return self.statsd.incr(*args, **kwargs) def decr(self, *args, **kwargs): return self.statsd.decr(*args, **kwargs) def gauge(self, *args, **kwargs): return self.statsd.gauge(*args, **kwargs) def setup_middleware(self): """Helper to configure/setup any Flask-StatsD middleware""" # Configure response time middleware (if desired) self.config.setdefault('STATSD_CONFIGURE_MIDDLEWARE', True) self.config.setdefault('STATSD_RESPONSE_METRIC_NAME', 'response.time') self.config.setdefault('STATSD_NUMBER_OF_REQUESTS_METRIC_NAME', 'request.api') self.config.setdefault('STATSD_RESPONSE_SAMPLE_RATE', 1) self.config.setdefault('STATSD_RESPONSE_AUTO_TAG', True) self.config.setdefault('STATSD_RESPONSE_ENDPOINT_TAG_FORMAT', 'endpoint_{0}') self.config.setdefault('STATSD_RESPONSE_METHOD_TAG_FORMAT', 'method_{0}') if self.config['STATSD_CONFIGURE_MIDDLEWARE']: self.app.before_request(self.before_request) self.app.after_request(self.after_request) def before_request(self): """ statsd middleware handle for before each request """ # Set the request start time g.flask_statsd_start_time = time.time() g.flask_statsd_request_tags = [] # Add some default request tags if self.config['STATSD_RESPONSE_AUTO_TAG']: self.add_request_tags([ # Endpoint tag self.config['STATSD_RESPONSE_ENDPOINT_TAG_FORMAT'].format(str(request.endpoint).lower()), # Method tag self.config['STATSD_RESPONSE_METHOD_TAG_FORMAT'].format(request.method.lower()), ]) # Send no of requests per second metric = '.'.join([self.config['STATSD_NUMBER_OF_REQUESTS_METRIC_NAME'], str(request.endpoint).lower(), request.method.lower()]) self.statsd.incr(metric, 1) def after_request(self, response): """ statsd middleware handler for after each request :param response: the response to be sent to the client :type response: ``flask.Response`` :rtype: ``flask.Response`` """ # Return early if we don't have the start time if not hasattr(g, 'flask_statsd_start_time'): return response # Get the response time for this request elapsed = time.time() - g.flask_statsd_start_time # Convert the elapsed time to milliseconds if they want them if self.use_ms: elapsed = int(round(1000 * elapsed)) # Add some additional response tags if self.config['STATSD_RESPONSE_AUTO_TAG']: self.add_request_tags(['status_code_%s' % (response.status_code, )]) metric = self.config['STATSD_RESPONSE_METRIC_NAME'] tags = self.get_request_tags() if tags: metric = ".".join([metric] + tags) # Emit our timing metric self.statsd.timing(metric, elapsed, rate=self.config['STATSD_RESPONSE_SAMPLE_RATE']) # We ALWAYS have to return the original response return response def get_request_tags(self): """ Get the current list of tags set for this request :rtype: list """ return getattr(g, 'flask_statsd_request_tags', []) def add_request_tags(self, tags): """ Add the provided list of tags to the tags stored for this request :param tags: tags to add to this requests tags :type tags: list :rtype: list """ # Get the current list of tags to append to # DEV: We use this method since ``self.get_request_tags`` will ensure that we get a list back current_tags = self.get_request_tags() # Append our new tags, and return the new full list of tags for this request g.flask_statsd_request_tags = current_tags + tags return g.flask_statsd_request_tags
class Worker(multiprocessing.Process): def __init__(self, worker_id, manager, redis_connection_params, sleep_time=0.1): self.manager = manager self.statsd_client = StatsClient(host=settings.STATSD_HOST, port=settings.STATSD_PORT, prefix=settings.STATSD_PREFIX) self.redis_connection_params = { k: v for k, v in redis_connection_params.iteritems() if k in ('host', 'db', 'password', 'port') } self.worker_id = None self.continue_working = True self.sleep_time = sleep_time self.child_pid = None self.current_job_id = None self.status = { 'jobs_count': 0, 'cancelled_jobs_count': 0, 'done_jobs_count': 0, 'updated_at': time.time(), 'started_at': time.time() } super(Worker, self).__init__(name="Worker") def set_title(self, title=None): base_title = "redash worker:%s" % self.worker_id if title: full_title = "%s - %s" % (base_title, title) else: full_title = base_title setproctitle.setproctitle(full_title) def run(self): self.worker_id = os.getpid() self.status['id'] = self.worker_id self.name = "Worker:%d" % self.worker_id self.manager.redis_connection.sadd('workers', self._key) self._save_status() self.set_title() logging.info("[%s] started.", self.name) signal.signal(signal.SIGINT, self._stop) signal.signal(signal.SIGTERM, self._stop) self._wait_for_jobs() def _stop(self, signum, frame): self.continue_working = False if self.current_job_id: job = Job.load(self.manager.redis_connection, self.current_job_id) if job: job.cancel() def _wait_for_jobs(self): while self.continue_working: job_id = self.manager.queue.pop() if job_id: self._update_status('jobs_count') logging.info("[%s] Processing %s", self.name, job_id) self._fork_and_process(job_id) if self.child_pid == 0: return else: time.sleep(self.sleep_time) def _update_status(self, counter): self.status['updated_at'] = time.time() self.status[counter] += 1 self._save_status() @property def _key(self): return 'worker:%s' % self.worker_id def _save_status(self): self.manager.redis_connection.hmset(self._key, self.status) def _fork_and_process(self, job_id): self.current_job_id = job_id self.child_pid = os.fork() if self.child_pid == 0: self.set_title("processing %s" % job_id) self._process(job_id) else: logging.info("[%s] Waiting for pid: %d", self.name, self.child_pid) try: _, status = os.waitpid(self.child_pid, 0) except OSError: logging.info("[%s] OSError while waiting for child to finish", self.name) # setting status to >0, so the job cleanup is triggered status = 1 self._update_status('done_jobs_count') job = Job.load(self.manager.redis_connection, job_id) if status > 0 and not job.is_finished(): self._update_status('cancelled_jobs_count') logging.info( "[%s] process interrupted and job %s hasn't finished; registering interruption in job", self.name, job_id) job.done(None, "Interrupted/Cancelled while running.") job.expire(settings.JOB_EXPIRY_TIME) logging.info("[%s] Finished Processing %s (pid: %d status: %d)", self.name, job_id, self.child_pid, status) self.child_pid = None self.current_job_id = None def _process(self, job_id): redis_connection = redis.StrictRedis(**self.redis_connection_params) job = Job.load(redis_connection, job_id) if job.is_finished(): logging.warning("[%s][%s] tried to process finished job.", self.name, job) return pid = os.getpid() job.processing(pid) logging.info("[%s][%s] running query...", self.name, job.id) start_time = time.time() self.set_title("running query %s" % job_id) logging.info("[%s][%s] Loading query runner (%s, %s)...", self.name, job.id, job.data_source_name, job.data_source_type) query_runner = get_query_runner(job.data_source_type, job.data_source_options) if getattr(query_runner, 'annotate_query', True): annotated_query = "/* Pid: %s, Job Id: %s, Query hash: %s, Priority: %s */ %s" % \ (pid, job.id, job.query_hash, job.priority, job.query) else: annotated_query = job.query # TODO: here's the part that needs to be forked, not all of the worker process... with self.statsd_client.timer( 'worker_{}.query_runner.{}.{}.run_time'.format( self.worker_id, job.data_source_type, job.data_source_name)): data, error = query_runner(annotated_query) run_time = time.time() - start_time logging.info("[%s][%s] query finished... data length=%s, error=%s", self.name, job.id, data and len(data), error) # TODO: it is possible that storing the data will fail, and we will need to retry # while we already marked the job as done query_result_id = None if not error: self.set_title("storing results %s" % job_id) query_result_id = self.manager.store_query_result( job.data_source_id, job.query, data, run_time, datetime.datetime.utcnow()) self.set_title("marking job as done %s" % job_id) job.done(query_result_id, error)
c = conn.cursor() util.create_schema(c) auth = util.authinfo(c) (q, option) = get_option(c, q) last_q = q.split(' ')[-1] if q.startswith('_'): # option process_option(c, q) elif q.startswith('+'): # add bookmark add_bookmark(c, q) elif last_q.startswith('#') and (':' not in q): # tag expansion pbsearch_tag(c, '', last_q[1:]) else: pbsearch_sql(c, option, q) util.closedb(conn) if __name__ == '__main__': try: statsd = StatsClient(host='g.jmjeong.com', port=8125, prefix='jmjeong.alfred.bookmark') with statsd.timer('main'): statsd.incr('launch') main() except: main()
from statsd import StatsClient from datetime import datetime from time import sleep statsd_client = StatsClient(host='metrics') ''' starting to include tags (remember: tags have to be strings) ''' for x in range(100,1000,100): print 'sleeping for {0} ms'.format(x) with statsd_client.timer('sd_timer,tag1=foo,x={}'.format(x)): sleep(float(x)/float(1000))
# Uptime pipe.gauge('os.uptime', uptime.uptime()) # Host clock offset try: response = ntplib.NTPClient().request('pool.ntp.org') except ntplib.NTPException: pass else: pipe.gauge('os.ntp.offset', response.offset) start = used = 0 metric = ns('watchtower', 'gathering') while True: start = int(time.time()) with statsd.timer(metric): loadavg() ram() network() os_status() used = int(time.time()) - start try: time.sleep(10 - used) # sleep for the remainder of the interval except IOError as exc: print "IOError on time.sleep(10 - %r): %s" % (used, exc) # Default sleep after an error time.sleep(8)
conn = util.opendb() c = conn.cursor() util.create_schema(c) auth = util.authinfo(c) (q,option) = get_option(c,q) last_q = q.split(' ')[-1] if q.startswith('_'): # option process_option(c,q) elif q.startswith('+'): # add bookmark add_bookmark(c,q) elif last_q.startswith('#') and (':' not in q): # tag expansion pbsearch_tag(c,'',last_q[1:]) else: pbsearch_sql(c,option,q) util.closedb(conn) if __name__ == '__main__': try: statsd = StatsClient(host='g.jmjeong.com', port=8125, prefix='jmjeong.alfred.bookmark') with statsd.timer('main'): statsd.incr('launch'); main() except: main()
class Client(object): def __init__(self, server, index='time_*', zero_fill=True, **kw): self.server = server self.session = requests.session() self.es = urljoin(self.server, index + '/_search') self.zero_fill = zero_fill # statsd settings statsd_host = kw.get('statsd.host', 'localhost') statsd_port = int(kw.get('statsd.port', 8125)) statsd_prefix = kw.get('statsd.prefix', 'monolith.client') self.statsd = StatsClient(host=statsd_host, port=statsd_port, prefix=statsd_prefix) def raw(self, query): with self.statsd.timer('elasticsearch-query'): res = self.session.get(self.es, data=json.dumps(query)) if res.status_code != 200: raise ValueError(res.content) # Get the JSON content. res = res.json if callable(res): res = res() self.statsd.incr('elasticsearch-call') if not isinstance(res, dict): raise ValueError(res) if 'errors' in res: raise ValueError(res['errors'][0]['description']) return res def __call__(self, field, start, end, interval=DAY, strict_range=False, **terms): if isinstance(interval, basestring): interval = _str2interval[interval.encode()] if isinstance(start, basestring): start = datetime.datetime.strptime(start.encode(), '%Y-%m-%d').toordinal() start = datetime.date.fromordinal(start) elif isinstance(start, datetime.datetime): start = start.date() if isinstance(end, basestring): end = datetime.datetime.strptime(end.encode(), '%Y-%m-%d').toordinal() end = datetime.date.fromordinal(end) elif isinstance(end, datetime.datetime): end = end.date() if interval == DAY: drange = util.iterdays(start, end) elif interval == WEEK: drange = util.iterweeks(start, end) elif interval == MONTH: drange = util.itermonths(start, end) else: drange = util.iteryears(start, end) # building the query start_date_str = start.strftime('%Y-%m-%d') end_date_str = end.strftime('%Y-%m-%d') if isinstance(interval, int): interval = _interval2str[interval] # XXX we'll see later if we want to provide a # nicer query interface # we need a facet query if strict_range: greater = "gt" lower = "lt" else: greater = "gte" lower = "lte" query = { "query": { "match_all": {}, }, "size": 0, # we aren't interested in the hits "facets": { "histo1": { "date_histogram": { "value_field": field, "interval": interval, "key_field": "date", }, "facet_filter": { "range": { "date": { greater: start_date_str, lower: end_date_str, } } } } } } if terms: range_ = query['facets']['histo1']['facet_filter']['range'] query['facets']['histo1']['facet_filter'] = { 'and': ([{'term': {k: v}} for k, v in terms.items()] + [{'range': range_}])} res = self.raw(query) counts = {} for entry in res['facets']['histo1']['entries']: time_ = entry['time'] / 1000.0 date_ = datetime.datetime.utcfromtimestamp(time_).date() if 'total' in entry: count = entry['total'] else: count = entry['count'] counts[date_] = count for date_ in drange: if strict_range and date_ in (start, end): continue if date_ in counts: yield {'count': counts[date_], 'date': date_} elif self.zero_fill: yield {'count': None, 'date': date_}
class Worker(multiprocessing.Process): def __init__(self, worker_id, manager, redis_connection_params, sleep_time=0.1): self.manager = manager self.statsd_client = StatsClient(host=settings.STATSD_HOST, port=settings.STATSD_PORT, prefix=settings.STATSD_PREFIX) self.redis_connection_params = {k: v for k, v in redis_connection_params.iteritems() if k in ('host', 'db', 'password', 'port')} self.worker_id = None self.continue_working = True self.sleep_time = sleep_time self.child_pid = None self.current_job_id = None self.status = { 'jobs_count': 0, 'cancelled_jobs_count': 0, 'done_jobs_count': 0, 'updated_at': time.time(), 'started_at': time.time() } super(Worker, self).__init__(name="Worker") def set_title(self, title=None): base_title = "redash worker:%s" % self.worker_id if title: full_title = "%s - %s" % (base_title, title) else: full_title = base_title setproctitle.setproctitle(full_title) def run(self): self.worker_id = os.getpid() self.status['id'] = self.worker_id self.name = "Worker:%d" % self.worker_id self.manager.redis_connection.sadd('workers', self._key) self._save_status() self.set_title() logging.info("[%s] started.", self.name) signal.signal(signal.SIGINT, self._stop) signal.signal(signal.SIGTERM, self._stop) self._wait_for_jobs() def _stop(self, signum, frame): self.continue_working = False if self.current_job_id: job = Job.load(self.manager.redis_connection, self.current_job_id) if job: job.cancel() def _wait_for_jobs(self): while self.continue_working: job_id = self.manager.queue.pop() if job_id: self._update_status('jobs_count') logging.info("[%s] Processing %s", self.name, job_id) self._fork_and_process(job_id) if self.child_pid == 0: return else: time.sleep(self.sleep_time) def _update_status(self, counter): self.status['updated_at'] = time.time() self.status[counter] += 1 self._save_status() @property def _key(self): return 'worker:%s' % self.worker_id def _save_status(self): self.manager.redis_connection.hmset(self._key, self.status) def _fork_and_process(self, job_id): self.current_job_id = job_id self.child_pid = os.fork() if self.child_pid == 0: self.set_title("processing %s" % job_id) self._process(job_id) else: logging.info("[%s] Waiting for pid: %d", self.name, self.child_pid) try: _, status = os.waitpid(self.child_pid, 0) except OSError: logging.info("[%s] OSError while waiting for child to finish", self.name) # setting status to >0, so the job cleanup is triggered status = 1 self._update_status('done_jobs_count') job = Job.load(self.manager.redis_connection, job_id) if status > 0 and not job.is_finished(): self._update_status('cancelled_jobs_count') logging.info("[%s] process interrupted and job %s hasn't finished; registering interruption in job", self.name, job_id) job.done(None, "Interrupted/Cancelled while running.") job.expire(settings.JOB_EXPIRY_TIME) logging.info("[%s] Finished Processing %s (pid: %d status: %d)", self.name, job_id, self.child_pid, status) self.child_pid = None self.current_job_id = None def _process(self, job_id): redis_connection = redis.StrictRedis(**self.redis_connection_params) job = Job.load(redis_connection, job_id) if job.is_finished(): logging.warning("[%s][%s] tried to process finished job.", self.name, job) return pid = os.getpid() job.processing(pid) logging.info("[%s][%s] running query...", self.name, job.id) start_time = time.time() self.set_title("running query %s" % job_id) logging.info("[%s][%s] Loading query runner (%s, %s)...", self.name, job.id, job.data_source_name, job.data_source_type) query_runner = get_query_runner(job.data_source_type, job.data_source_options) if getattr(query_runner, 'annotate_query', True): annotated_query = "/* Pid: %s, Job Id: %s, Query hash: %s, Priority: %s */ %s" % \ (pid, job.id, job.query_hash, job.priority, job.query) else: annotated_query = job.query # TODO: here's the part that needs to be forked, not all of the worker process... with self.statsd_client.timer('worker_{}.query_runner.{}.{}.run_time'.format(self.worker_id, job.data_source_type, job.data_source_name)): data, error = query_runner(annotated_query) run_time = time.time() - start_time logging.info("[%s][%s] query finished... data length=%s, error=%s", self.name, job.id, data and len(data), error) # TODO: it is possible that storing the data will fail, and we will need to retry # while we already marked the job as done query_result_id = None if not error: self.set_title("storing results %s" % job_id) query_result_id = self.manager.store_query_result(job.data_source_id, job.query, data, run_time, datetime.datetime.utcnow()) self.set_title("marking job as done %s" % job_id) job.done(query_result_id, error)
class StatsdMonitor(object): def __init__(self, broker, interval=1): # self.interval = interval self.state = app.events.State() self.statsd_conn = StatsClient(host='localhost', port=8125) self.broker_conn = BrokerConnection(broker) self.timers_list = [] # monitor the task and status of worker with functions def run_loop(self): while True: try: with self.broker_conn as conn: recv = EventReceiver(conn, handlers={ 'task-sent': self.on_task_sent, 'task-failed': self.on_task_failed, 'task-retried': self.on_task_retried, 'task-started': self.on_task_started, 'task-succeeded': self.on_task_succeeded, 'task-received': self.on_task_received, 'task-rejected': self.on_task_rejected, 'task-revoked': self.on_task_revoked, 'worker-online': self.on_worker_online, 'worker-heartbeat': self.on_worker_heartbeat, 'worker-offline': self.on_worker_offline, }) recv.capture(limit=None, timeout=None, wakeup=True) except (KeyboardInterrupt, SystemExit): raise except Exception: raise # time.sleep(self.interval) # all about the tasks def on_task_sent(self, event): # TODO self.state.event(event) task = self.state.tasks.get(event['uuid']) self.statsd_conn.incr('tasks.sent') def on_task_received(self, event): # TODO self.state.event(event) task = self.state.tasks.get(event['uuid']) self.statsd_conn.incr('tasks.received') def on_task_started(self, event): self.state.event(event) task = self.state.tasks.get(event['uuid']) logger.info('Task {}[{}] started'.format(task.name, task.uuid)) self.statsd_conn.incr('tasks.started') mark = 'task.{}.recorder'.format(task.uuid) self.timer_start(mark) def on_task_succeeded(self, event): self.state.event(event) task = self.state.tasks.get(event['uuid']) logger.info('Task {}[{}] succeeded'.format(task.name, task.uuid)) self.statsd_conn.incr('tasks.succeeded') mark = 'task.{}.recorder'.format(task.uuid) self.timer_stop(mark) def on_task_failed(self, event): # TODO self.state.event(event) task = self.state.tasks.get(event['uuid']) logger.warning('Task {}[{}] failed'.format(task.name, task.uuid)) self.statsd_conn.incr('tasks.failed') def on_task_retried(self, event): # TODO self.state.event(event) task = self.state.tasks.get(event['uuid']) logger.warning('Task {}[{}] retried'.format(task.name, task.uuid)) self.statsd_conn.incr('tasks.retried') def on_task_rejected(self, event): # TODO self.state.event(event) task = self.state.tasks.get(event['uuid']) def on_task_revoked(self, event): # TODO self.state.event(event) task = self.state.tasks.get(event['uuid']) # all about the status of the workers def on_worker_online(self, event): # TODO self.state.event(event) worker = self.state.workers.get(event['hostname']) mark = 'worker.{}.recorder'.format(worker.hostname) self.timer_start(mark) def on_worker_heartbeat(self, event): self.state.event(event) worker = self.state.workers.get(event['hostname']) key_pro = 'worker.{}.processed'.format(worker.hostname) key_act = 'worker.{}.active'.format(worker.hostname) if worker.processed is None: worker.processed = 0 if worker.active is None: worker.active = 0 self.statsd_conn.gauge(key_pro, worker.processed) self.statsd_conn.gauge(key_act, worker.active) def on_worker_offline(self, event): # TODO self.state.event(event) worker = self.state.workers.get(event['hostname']) mark = 'worker.{}.recorder'.format(worker.hostname) self.timer_stop(mark) # statsd timer record start def timer_start(self, mark): timer = self.statsd_conn.timer(mark) timer.start() self.timers_list.append(timer) # statsd timer record stop def timer_stop(self, mark): for timer in self.timers_list: if timer.stat == mark: timer.stop() self.timers_list.remove(timer)
from statsd import StatsClient from datetime import datetime from time import sleep statsd_client = StatsClient(host='metrics') for x in range(100,1000,100): # start of loop print 'sleeping for {0} ms'.format(x) # print sleep time with statsd_client.timer('sd_timer'): # begin statsd timer sleep(float(x)/float(1000)) # sleep X milliseconds
with open(CONFIG_FILE, 'r') as f: data = json.load(f) REDIS_SERVER_HOST = data['redis']['redisServerHost'] REDIS_SERVER_PORT = int(data['redis']['redisServerPort']) REDIS_NEWS_EXPIRE_IN_SECONDS = int(data['redis']['newsExpireInSeconds']) NEWS_LIMIT = int(data['news']['newsLimit']) NEWS_PER_PAGE_SIZE = int(data['news']['newsPageSize']) NEWS_DB_COLLECTION = data['mongoDb']['newsMongoDbCollection'] CLICKS_DB_COLLECTION = data['mongoDb']['clicksMongoDbCollection'] LOG_CLICKS_TASK_QUEUE_URL = data['queue']['logClicksTaskQueueUrl'] LOG_CLICKS_TASK_QUEUE_NAME = data['queue']['logClicksTaskQueueName'] redis_client = redis.StrictRedis(REDIS_SERVER_HOST, REDIS_SERVER_PORT) CloudAMQPClient = CloudAMQPClient(LOG_CLICKS_TASK_QUEUE_URL, LOG_CLICKS_TASK_QUEUE_NAME) statsd = StatsClient() news_timer = statsd.timer('news') def isclose(a, b, rel_tol=1e-09, abs_tol=0.0): return abs(a-b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol) def getNewssummaryForuser(userId, pageNum): news_timer.start() print "get news for user: %s @ page %s" % (userId, pageNum) statsd.incr("user.news") pageNum = int(pageNum) startIndex = (pageNum - 1) * NEWS_PER_PAGE_SIZE endIndex = pageNum * NEWS_PER_PAGE_SIZE sliced_news = [] #get preference for user pref_dict = news_recommendation_service_client.getPreferenceForuser(userId)
from statsd import StatsClient statsd = StatsClient() foo_timer = statsd.timer('foo') foo_timer.start() # Do something fun. foo_timer.stop()