def _evaluate(experiment: EvExperiment, dao: Dao, statsd: StatsClient):
try:
with statsd.timer('timing.evaluation'):
_logger.info(f'Evaluating experiment [{experiment.id}]')
_logger.debug(
f'Loading goals for experiment [{experiment.id}]')
with statsd.timer('timing.query'):
goals = dao.get_agg_goals(experiment).sort_values(
['exp_variant_id', 'goal'])
_logger.info(
f'Retrieved {len(goals)} goals in experiment [{experiment.id}]'
)
with statsd.timer('timing.stats'):
evaluation = experiment.evaluate_agg(goals)
statsd.incr('evaluations')
_logger.info((
f'Evaluation of experiment [{experiment.id}] finished with evaluation'
f' of {evaluation.metrics.metric_id.nunique()} '
f'metrics and {evaluation.checks.check_id.nunique()} checks'
))
return Result.from_evaluation(experiment, evaluation)
except Exception as e:
_logger.error(
f'Cannot evaluate experiment [{experiment.id}] because of {e}')
_logger.exception(e)
statsd.incr('errors.experiment')
raise HTTPException(
status_code=500,
detail=
f'Cannot evaluate experiment [{experiment.id}] because of {e}',
)
class StatsdMetrics(Metrics):
def __init__(self, host='localhost', port=8125, prefix=None):
self.statsd = StatsClient(host, port, prefix)
def fanout_timer(self, feed_class):
return self.statsd.timer('%s.fanout_latency' % feed_class.__name__)
def feed_reads_timer(self, feed_class):
return self.statsd.timer('%s.read_latency' % feed_class.__name__)
def on_feed_read(self, feed_class, activities_count):
self.statsd.incr('%s.reads' % feed_class.__name__, activities_count)
def on_feed_write(self, feed_class, activities_count):
self.statsd.incr('%s.writes' % feed_class.__name__, activities_count)
def on_feed_remove(self, feed_class, activities_count):
self.statsd.incr('%s.deletes' % feed_class.__name__, activities_count)
def on_fanout(self, feed_class, operation, activities_count=1):
metric = (feed_class.__name__, operation.__name__)
self.statsd.incr('%s.fanout.%s' % metric, activities_count)
def on_activity_published(self):
self.statsd.incr('activities.published')
def on_activity_removed(self):
self.statsd.incr('activities.removed')
class StatsdMetrics(Metrics):
def __init__(self, host='localhost', port=8125, prefix=None):
self.statsd = StatsClient(host, port, prefix)
def fanout_timer(self, feed_class):
return self.statsd.timer('%s.fanout_latency' % feed_class.__name__)
def feed_reads_timer(self, feed_class):
return self.statsd.timer('%s.read_latency' % feed_class.__name__)
def on_feed_read(self, feed_class, activities_count):
self.statsd.incr('%s.reads' % feed_class.__name__, activities_count)
def on_feed_write(self, feed_class, activities_count):
self.statsd.incr('%s.writes' % feed_class.__name__, activities_count)
def on_feed_remove(self, feed_class, activities_count):
self.statsd.incr('%s.deletes' % feed_class.__name__, activities_count)
def on_fanout(self, feed_class, operation, activities_count=1):
metric = (feed_class.__name__, operation.__name__)
self.statsd.incr('%s.fanout.%s' % metric, activities_count)
def on_activity_published(self):
self.statsd.incr('activities.published')
def on_activity_removed(self):
self.statsd.incr('activities.removed')
class StatsdWrapper:
"""Simple wrapper around the statsd client."""
statsd = None
def __init__(self, host, port, prefix):
if host:
self.statsd = StatsClient(
host=host,
port=port,
prefix=prefix,
)
def incr(self, *args):
if self.statsd:
self.statsd.incr(*args)
def decr(self, *args):
if self.statsd:
self.statsd.decr(*args)
def gauge(self, *args):
if self.statsd:
self.statsd.gauge(*args)
def timing(self, *args):
if self.statsd:
self.statsd.timing(*args)
def timer(self, *args):
if self.statsd:
self.statsd.timer(*args)
def set(self, *args):
if self.statsd:
self.statsd.set(*args)
class FlaskStat(object):
_xstat_title = None
_xstat_host = None
_xstat_port = None
_stat_client = None
def __init__(self, app=None):
super(FlaskStat, self).__init__()
if app:
self.init_app(app)
def init_app(self, app):
from flask import request, g
"""
绑定app
"""
self._xstat_title = app.config.get('XSTAT_TITLE')
self._xstat_host = app.config.get('XSTAT_HOST')
self._xstat_port = app.config.get('XSTAT_PORT') or constants.XSTAT_PORT
self._stat_client = StatsClient(host=self._xstat_host, port=self._xstat_port)
@app.before_request
@catch_exc
def prepare_stat():
if not request.endpoint:
return
g.xstat_timers = []
g.xstat_timers.append(
self._stat_client.timer('.'.join([
self._xstat_title,
'endpoint',
request.endpoint,
])
)
)
g.xstat_timers.append(
self._stat_client.timer('.'.join([
self._xstat_title,
'all',
])
)
)
for stat in g.xstat_timers:
stat.start()
@app.teardown_request
@catch_exc
def send_stat(exc):
if not hasattr(g, 'xstat_timers'):
return
for stat in g.xstat_timers:
stat.stop()
class MapleStat(object):
_xstat_title = None
_xstat_host = None
_xstat_port = None
_stat_client = None
def __init__(self, app=None, config=None):
super(MapleStat, self).__init__()
if app:
self.init_app(app, config)
def init_app(self, app, config):
"""
绑定app
"""
self._xstat_title = config.get('XSTAT_TITLE')
self._xstat_host = config.get('XSTAT_HOST')
self._xstat_port = config.get('XSTAT_PORT') or constants.XSTAT_PORT
self._stat_client = StatsClient(host=self._xstat_host, port=self._xstat_port)
@app.before_request
@catch_exc
def prepare_stat(request):
if not request.endpoint:
return
request.xstat_timers = []
request.xstat_timers.append(
self._stat_client.timer('.'.join([
self._xstat_title,
'endpoint',
request.endpoint,
])
)
)
request.xstat_timers.append(
self._stat_client.timer('.'.join([
self._xstat_title,
'all',
])
)
)
for stat in request.xstat_timers:
stat.start()
@app.after_request
@catch_exc
def send_stat(request, exc):
if not hasattr(request, 'xstat_timers'):
return
for stat in request.xstat_timers:
stat.stop()
class DjangoStat(MiddlewareMixin):
_xstat_title = None
_xstat_host = None
_xstat_port = None
_stat_client = None
def __init__(self, *args, **kwargs):
from django.conf import settings
super(DjangoStat, self).__init__(*args, **kwargs)
self._xstat_title = getattr(settings, 'XSTAT_TITLE', None)
self._xstat_host = getattr(settings, 'XSTAT_HOST', None)
self._xstat_port = getattr(settings, 'XSTAT_PORT', None) or constants.XSTAT_PORT
self._stat_client = StatsClient(host=self._xstat_host, port=self._xstat_port)
@catch_exc
def process_view(self, request, view_func, view_args, view_kwargs):
"""
request.resolver_match.url_name 在process_view才可以取到
:return:
"""
request.xstat_timers = []
request.xstat_timers.append(
self._stat_client.timer('.'.join([
self._xstat_title,
'endpoint',
request.resolver_match.url_name,
])
)
)
request.xstat_timers.append(
self._stat_client.timer('.'.join([
self._xstat_title,
'all',
])
)
)
for stat in request.xstat_timers:
stat.start()
@catch_exc
def process_response(self, request, response):
"""
无论是否抛出异常，都会执行这一步
"""
if not hasattr(request, 'xstat_timers'):
return response
for stat in request.xstat_timers:
stat.stop()
return response
class FlaskStat(object):
_xstat_title = None
_xstat_host = None
_xstat_port = None
_stat_client = None
def __init__(self, app=None):
super(FlaskStat, self).__init__()
if app:
self.init_app(app)
def init_app(self, app):
from flask import request, g
"""
绑定app
"""
self._xstat_title = app.config.get('XSTAT_TITLE')
self._xstat_host = app.config.get('XSTAT_HOST')
self._xstat_port = app.config.get('XSTAT_PORT') or constants.XSTAT_PORT
self._stat_client = StatsClient(host=self._xstat_host,
port=self._xstat_port)
@app.before_request
@catch_exc
def prepare_stat():
if not request.endpoint:
return
g.xstat_timers = []
g.xstat_timers.append(
self._stat_client.timer('.'.join([
self._xstat_title,
'endpoint',
request.endpoint,
])))
g.xstat_timers.append(
self._stat_client.timer('.'.join([
self._xstat_title,
'all',
])))
for stat in g.xstat_timers:
stat.start()
@app.teardown_request
@catch_exc
def send_stat(exc):
if not hasattr(g, 'xstat_timers'):
return
for stat in g.xstat_timers:
stat.stop()
class DjangoStat(object):
_xstat_title = None
_xstat_host = None
_xstat_port = None
_stat_client = None
def __init__(self):
from django.conf import settings
super(DjangoStat, self).__init__()
self._xstat_title = getattr(settings, 'XSTAT_TITLE', None)
self._xstat_host = getattr(settings, 'XSTAT_HOST', None)
self._xstat_port = getattr(settings, 'XSTAT_PORT',
None) or constants.XSTAT_PORT
self._stat_client = StatsClient(host=self._xstat_host,
port=self._xstat_port)
@catch_exc
def process_view(self, request, view_func, view_args, view_kwargs):
"""
request.resolver_match.url_name 在process_view才可以取到
:return:
"""
request.xstat_timers = []
request.xstat_timers.append(
self._stat_client.timer('.'.join([
self._xstat_title,
'endpoint',
request.resolver_match.url_name,
])))
request.xstat_timers.append(
self._stat_client.timer('.'.join([
self._xstat_title,
'all',
])))
for stat in request.xstat_timers:
stat.start()
@catch_exc
def process_response(self, request, response):
"""
无论是否抛出异常，都会执行这一步
"""
if not hasattr(request, 'xstat_timers'):
return response
for stat in request.xstat_timers:
stat.stop()
return response
class MapleStat(object):
_xstat_title = None
_xstat_host = None
_xstat_port = None
_stat_client = None
def __init__(self, app=None, config=None):
super(MapleStat, self).__init__()
if app:
self.init_app(app, config)
def init_app(self, app, config):
"""
绑定app
"""
self._xstat_title = config.get('XSTAT_TITLE')
self._xstat_host = config.get('XSTAT_HOST')
self._xstat_port = config.get('XSTAT_PORT') or constants.XSTAT_PORT
self._stat_client = StatsClient(host=self._xstat_host,
port=self._xstat_port)
@app.before_request
@catch_exc
def prepare_stat(request):
if not request.endpoint:
return
request.xstat_timers = []
request.xstat_timers.append(
self._stat_client.timer('.'.join([
self._xstat_title,
'endpoint',
request.endpoint,
])))
request.xstat_timers.append(
self._stat_client.timer('.'.join([
self._xstat_title,
'all',
])))
for stat in request.xstat_timers:
stat.start()
@app.after_request
@catch_exc
def send_stat(request, exc):
if not hasattr(request, 'xstat_timers'):
return
for stat in request.xstat_timers:
stat.stop()
class _Statsd(object):
def __init__(self, config):
if config.get('datadog', True):
initialize(statsd_host=config['host'],
statsd_port=config['port'],
prefix=config['prefix'])
self.datadog = True
self._statsd = statsd
else:
self.datadog = False
self._statsd = StatsClient(config['host'],
config['port'],
config['prefix'])
def incr(self, metric, count=1, rate=1, **kw):
if self.datadog:
return self._statsd.increment(metric, value=count,
sample_rate=rate, **kw)
else:
return self._statsd.incr(metric, count=count, rate=rate)
def timer(self, metric, rate=1, **kw):
if self.datadog:
return self._statsd.timed(metric, sample_rate=rate, **kw)
else:
return self._statsd.timer(metric, rate=rate)
class _Statsd(object):
def __init__(self, config):
if config.get('datadog', True):
initialize(statsd_host=config['host'],
statsd_port=config['port'],
prefix=config['prefix'])
self.datadog = True
self._statsd = statsd
else:
self.datadog = False
self._statsd = StatsClient(config['host'], config['port'],
config['prefix'])
def incr(self, metric, count=1, rate=1, **kw):
if self.datadog:
return self._statsd.increment(metric,
value=count,
sample_rate=rate,
**kw)
else:
return self._statsd.incr(metric, count=count, rate=rate)
def timer(self, metric, rate=1, **kw):
if self.datadog:
return self._statsd.timed(metric, sample_rate=rate, **kw)
else:
return self._statsd.timer(metric, rate=rate)
def time_stack_list(username, password, tenant, auth_url, heat_url, region,
statsd_server):
keystone = keystone_client(username=username, password=password,
tenant_name=tenant, auth_url=auth_url)
token = keystone.auth_token
heat = heat_client('1', endpoint=heat_url, region_name=region, token=token)
statsd = StatsClient(host=statsd_server)
with statsd.timer('uptime.{}'.format(region)):
list(heat.stacks.list())
class StatsD(object):
def __init__(self, app=None, config=None):
self.config = None
self.statsd = None
if app is not None:
self.init_app(app)
else:
self.app = None
def init_app(self, app, config=None):
if config is not None:
self.config = config
elif self.config is None:
self.config = app.config
self.config.setdefault('STATSD_HOST', 'localhost')
self.config.setdefault('STATSD_PORT', 8125)
self.config.setdefault('STATSD_PREFIX', None)
self.app = app
self.statsd = StatsClient(self.config['STATSD_HOST'],
self.config['STATSD_PORT'],
self.config['STATSD_PREFIX'])
def timer(self, *args, **kwargs):
return self.statsd.timer(*args, **kwargs)
def timing(self, *args, **kwargs):
return self.statsd.timing(*args, **kwargs)
def incr(self, *args, **kwargs):
return self.statsd.incr(*args, **kwargs)
def decr(self, *args, **kwargs):
return self.statsd.decr(*args, **kwargs)
def gauge(self, *args, **kwargs):
return self.statsd.gauge(*args, **kwargs)
def set(self, *args, **kwargs):
return self.statsd.set(*args, **kwargs)
class StatsD(object):
def __init__(self, app=None, config=None):
self.config = None
self.statsd = None
if app is not None:
self.init_app(app)
else:
self.app = None
def init_app(self, app, config=None):
if config is not None:
self.config = config
elif self.config is None:
self.config = app.config
self.config.setdefault('STATSD_HOST', 'localhost')
self.config.setdefault('STATSD_PORT', 8125)
self.config.setdefault('STATSD_PREFIX', None)
self.app = app
self.statsd = StatsClient(self.config['STATSD_HOST'],
self.config['STATSD_PORT'], self.config['STATSD_PREFIX'])
def timer(self, *args, **kwargs):
return self.statsd.timer(*args, **kwargs)
def timing(self, *args, **kwargs):
return self.statsd.timing(*args, **kwargs)
def incr(self, *args, **kwargs):
return self.statsd.incr(*args, **kwargs)
def decr(self, *args, **kwargs):
return self.statsd.decr(*args, **kwargs)
def gauge(self, *args, **kwargs):
return self.statsd.gauge(*args, **kwargs)
def set(self, *args, **kwargs):
return self.statsd.set(*args, **kwargs)
class StatsD(object):
def __init__(self, app=None, config=None):
self.config = None
self.statsd = None
if app is not None:
self.init_app(app)
else:
self.app = None
def init_app(self, app, config=None):
if config is not None:
self.config = config
elif self.config is None:
self.config = app.config
self.config.setdefault("STATSD_HOST", "localhost")
self.config.setdefault("STATSD_PORT", 8125)
self.config.setdefault("STATSD_PREFIX", None)
self.app = app
self.statsd = StatsClient(
host=self.config["STATSD_HOST"], port=self.config["STATSD_PORT"], prefix=self.config["STATSD_PREFIX"]
)
def timer(self, *args, **kwargs):
return self.statsd.timer(*args, **kwargs)
def timing(self, *args, **kwargs):
return self.statsd.timing(*args, **kwargs)
def incr(self, *args, **kwargs):
return self.statsd.incr(*args, **kwargs)
def decr(self, *args, **kwargs):
return self.statsd.decr(*args, **kwargs)
def gauge(self, *args, **kwargs):
return self.statsd.gauge(*args, **kwargs)
class Client(object):
def __init__(self, server, zero_fill=True, **kw):
self.server = server.rstrip('/')
self.session = requests.session()
# getting monolith info
info = self.session.get(server).json
if callable(info):
info = info()
self.es = self.server + info['es_endpoint']
self.fields = info['fields']
self.zero_fill = zero_fill
# statsd settings
statsd_host = kw.get('statsd.host', 'localhost')
statsd_port = int(kw.get('statsd.port', 8125))
statsd_prefix = kw.get('statsd.prefix', 'monolith.client')
self.statsd = StatsClient(host=statsd_host, port=statsd_port,
prefix=statsd_prefix)
def __call__(self, field, start, end, interval=DAY, strict_range=False,
**terms):
if isinstance(interval, basestring):
interval = _str2interval[interval.encode()]
if isinstance(start, basestring):
start = datetime.datetime.strptime(start.encode(),
'%Y-%m-%d').toordinal()
start = datetime.date.fromordinal(start)
end = datetime.datetime.strptime(end.encode(),
'%Y-%m-%d').toordinal()
end = datetime.date.fromordinal(end)
if interval == DAY:
drange = util.iterdays(start, end)
elif interval == WEEK:
drange = util.iterweeks(start, end)
elif interval == MONTH:
drange = util.itermonths(start, end)
else:
drange = util.iteryears(start, end)
# building the query
start_date_str = start.strftime('%Y-%m-%d')
end_date_str = end.strftime('%Y-%m-%d')
if isinstance(interval, int):
interval = _interval2str[interval]
# XXX we'll see later if we want to provide a
# nicer query interface
# we need a facet query
if strict_range:
greater = "gt"
lower = "lt"
else:
greater = "gte"
lower = "lte"
query = {
"query": {
"match_all": {},
},
"size": 0, # we aren't interested in the hits
"facets": {
"histo1": {
"date_histogram": {
"value_field": field,
"interval": interval,
"key_field": "date",
},
"facet_filter": {
"range": {
"date": {
greater: start_date_str,
lower: end_date_str,
}
}
}
}
}
}
if len(terms) > 0:
term = {}
for key, value in terms.items():
term[key] = value
range_ = query['facets']['histo1']['facet_filter']['range']
filter_ = {'and': [{'term': term},
{'range': range_}]}
query['facets']['histo1']['facet_filter'] = filter_
with self.statsd.timer('elasticsearch-query'):
res = self.session.post(self.es, data=json.dumps(query))
if res.status_code != 200:
raise ValueError(res.content)
# getting the JSON content
res = res.json
if callable(res):
res = res()
# statsd calls
self.statsd.incr('elasticsearch-call')
if not isinstance(res, dict):
raise ValueError(res)
if 'errors' in res:
raise ValueError(res['errors'][0]['description'])
dates = set()
for entry in res['facets']['histo1']['entries']:
time_ = entry['time'] / 1000.0
date_ = datetime.datetime.utcfromtimestamp(time_).date()
if 'total' in entry:
count = entry['total']
else:
count = entry['count']
if date_ not in dates:
dates.add(date_)
yield {'count': count, 'date': date_}
if self.zero_fill:
# yielding zeros
for date_ in drange:
if strict_range and date_ in (start, end):
continue
if date_ not in dates:
yield {'count': 0, 'date': date_}
class StatsD(object):
def __init__(self, app=None, config=None):
self.config = None
self.statsd = None
if app is not None:
self.init_app(app, config=config)
else:
self.app = None
def init_app(self, app, config=None):
if config is not None:
self.config = config
elif self.config is None:
self.config = app.config
self.config.setdefault('STATSD_HOST', 'localhost')
self.config.setdefault('STATSD_PORT', 8125)
self.config.setdefault('STATSD_PREFIX', None)
self.app = app
self.statsd = StatsClient(self.config['STATSD_HOST'],
self.config['STATSD_PORT'], self.config['STATSD_PREFIX'])
self.use_ms=self.config.get('STATSD_USEMS', True)
# Configure any of our middleware
self.setup_middleware()
def timer(self, *args, **kwargs):
return self.statsd.timer(*args, **kwargs)
def timing(self, *args, **kwargs):
return self.statsd.timing(*args, **kwargs)
def incr(self, *args, **kwargs):
return self.statsd.incr(*args, **kwargs)
def decr(self, *args, **kwargs):
return self.statsd.decr(*args, **kwargs)
def gauge(self, *args, **kwargs):
return self.statsd.gauge(*args, **kwargs)
def setup_middleware(self):
"""Helper to configure/setup any Flask-StatsD middleware"""
# Configure response time middleware (if desired)
self.config.setdefault('STATSD_CONFIGURE_MIDDLEWARE', True)
self.config.setdefault('STATSD_RESPONSE_METRIC_NAME', 'response.time')
self.config.setdefault('STATSD_NUMBER_OF_REQUESTS_METRIC_NAME', 'request.api')
self.config.setdefault('STATSD_RESPONSE_SAMPLE_RATE', 1)
self.config.setdefault('STATSD_RESPONSE_AUTO_TAG', True)
self.config.setdefault('STATSD_RESPONSE_ENDPOINT_TAG_FORMAT', 'endpoint_{0}')
self.config.setdefault('STATSD_RESPONSE_METHOD_TAG_FORMAT', 'method_{0}')
if self.config['STATSD_CONFIGURE_MIDDLEWARE']:
self.app.before_request(self.before_request)
self.app.after_request(self.after_request)
def before_request(self):
"""
statsd middleware handle for before each request
"""
# Set the request start time
g.flask_statsd_start_time = time.time()
g.flask_statsd_request_tags = []
# Add some default request tags
if self.config['STATSD_RESPONSE_AUTO_TAG']:
self.add_request_tags([
# Endpoint tag
self.config['STATSD_RESPONSE_ENDPOINT_TAG_FORMAT'].format(str(request.endpoint).lower()),
# Method tag
self.config['STATSD_RESPONSE_METHOD_TAG_FORMAT'].format(request.method.lower()),
])
# Send no of requests per second
metric = '.'.join([self.config['STATSD_NUMBER_OF_REQUESTS_METRIC_NAME'],
str(request.endpoint).lower(), request.method.lower()])
self.statsd.incr(metric, 1)
def after_request(self, response):
"""
statsd middleware handler for after each request
:param response: the response to be sent to the client
:type response: ``flask.Response``
:rtype: ``flask.Response``
"""
# Return early if we don't have the start time
if not hasattr(g, 'flask_statsd_start_time'):
return response
# Get the response time for this request
elapsed = time.time() - g.flask_statsd_start_time
# Convert the elapsed time to milliseconds if they want them
if self.use_ms:
elapsed = int(round(1000 * elapsed))
# Add some additional response tags
if self.config['STATSD_RESPONSE_AUTO_TAG']:
self.add_request_tags(['status_code_%s' % (response.status_code, )])
metric = self.config['STATSD_RESPONSE_METRIC_NAME']
tags = self.get_request_tags()
if tags:
metric = ".".join([metric] + tags)
# Emit our timing metric
self.statsd.timing(metric,
elapsed, rate=self.config['STATSD_RESPONSE_SAMPLE_RATE'])
# We ALWAYS have to return the original response
return response
def get_request_tags(self):
"""
Get the current list of tags set for this request
:rtype: list
"""
return getattr(g, 'flask_statsd_request_tags', [])
def add_request_tags(self, tags):
"""
Add the provided list of tags to the tags stored for this request
:param tags: tags to add to this requests tags
:type tags: list
:rtype: list
"""
# Get the current list of tags to append to
# DEV: We use this method since ``self.get_request_tags`` will ensure that we get a list back
current_tags = self.get_request_tags()
# Append our new tags, and return the new full list of tags for this request
g.flask_statsd_request_tags = current_tags + tags
return g.flask_statsd_request_tags
class Worker(multiprocessing.Process):
def __init__(self,
worker_id,
manager,
redis_connection_params,
sleep_time=0.1):
self.manager = manager
self.statsd_client = StatsClient(host=settings.STATSD_HOST,
port=settings.STATSD_PORT,
prefix=settings.STATSD_PREFIX)
self.redis_connection_params = {
k: v
for k, v in redis_connection_params.iteritems()
if k in ('host', 'db', 'password', 'port')
}
self.worker_id = None
self.continue_working = True
self.sleep_time = sleep_time
self.child_pid = None
self.current_job_id = None
self.status = {
'jobs_count': 0,
'cancelled_jobs_count': 0,
'done_jobs_count': 0,
'updated_at': time.time(),
'started_at': time.time()
}
super(Worker, self).__init__(name="Worker")
def set_title(self, title=None):
base_title = "redash worker:%s" % self.worker_id
if title:
full_title = "%s - %s" % (base_title, title)
else:
full_title = base_title
setproctitle.setproctitle(full_title)
def run(self):
self.worker_id = os.getpid()
self.status['id'] = self.worker_id
self.name = "Worker:%d" % self.worker_id
self.manager.redis_connection.sadd('workers', self._key)
self._save_status()
self.set_title()
logging.info("[%s] started.", self.name)
signal.signal(signal.SIGINT, self._stop)
signal.signal(signal.SIGTERM, self._stop)
self._wait_for_jobs()
def _stop(self, signum, frame):
self.continue_working = False
if self.current_job_id:
job = Job.load(self.manager.redis_connection, self.current_job_id)
if job:
job.cancel()
def _wait_for_jobs(self):
while self.continue_working:
job_id = self.manager.queue.pop()
if job_id:
self._update_status('jobs_count')
logging.info("[%s] Processing %s", self.name, job_id)
self._fork_and_process(job_id)
if self.child_pid == 0:
return
else:
time.sleep(self.sleep_time)
def _update_status(self, counter):
self.status['updated_at'] = time.time()
self.status[counter] += 1
self._save_status()
@property
def _key(self):
return 'worker:%s' % self.worker_id
def _save_status(self):
self.manager.redis_connection.hmset(self._key, self.status)
def _fork_and_process(self, job_id):
self.current_job_id = job_id
self.child_pid = os.fork()
if self.child_pid == 0:
self.set_title("processing %s" % job_id)
self._process(job_id)
else:
logging.info("[%s] Waiting for pid: %d", self.name, self.child_pid)
try:
_, status = os.waitpid(self.child_pid, 0)
except OSError:
logging.info("[%s] OSError while waiting for child to finish",
self.name)
# setting status to >0, so the job cleanup is triggered
status = 1
self._update_status('done_jobs_count')
job = Job.load(self.manager.redis_connection, job_id)
if status > 0 and not job.is_finished():
self._update_status('cancelled_jobs_count')
logging.info(
"[%s] process interrupted and job %s hasn't finished; registering interruption in job",
self.name, job_id)
job.done(None, "Interrupted/Cancelled while running.")
job.expire(settings.JOB_EXPIRY_TIME)
logging.info("[%s] Finished Processing %s (pid: %d status: %d)",
self.name, job_id, self.child_pid, status)
self.child_pid = None
self.current_job_id = None
def _process(self, job_id):
redis_connection = redis.StrictRedis(**self.redis_connection_params)
job = Job.load(redis_connection, job_id)
if job.is_finished():
logging.warning("[%s][%s] tried to process finished job.",
self.name, job)
return
pid = os.getpid()
job.processing(pid)
logging.info("[%s][%s] running query...", self.name, job.id)
start_time = time.time()
self.set_title("running query %s" % job_id)
logging.info("[%s][%s] Loading query runner (%s, %s)...", self.name,
job.id, job.data_source_name, job.data_source_type)
query_runner = get_query_runner(job.data_source_type,
job.data_source_options)
if getattr(query_runner, 'annotate_query', True):
annotated_query = "/* Pid: %s, Job Id: %s, Query hash: %s, Priority: %s */ %s" % \
(pid, job.id, job.query_hash, job.priority, job.query)
else:
annotated_query = job.query
# TODO: here's the part that needs to be forked, not all of the worker process...
with self.statsd_client.timer(
'worker_{}.query_runner.{}.{}.run_time'.format(
self.worker_id, job.data_source_type,
job.data_source_name)):
data, error = query_runner(annotated_query)
run_time = time.time() - start_time
logging.info("[%s][%s] query finished... data length=%s, error=%s",
self.name, job.id, data and len(data), error)
# TODO: it is possible that storing the data will fail, and we will need to retry
# while we already marked the job as done
query_result_id = None
if not error:
self.set_title("storing results %s" % job_id)
query_result_id = self.manager.store_query_result(
job.data_source_id, job.query, data, run_time,
datetime.datetime.utcnow())
self.set_title("marking job as done %s" % job_id)
job.done(query_result_id, error)
c = conn.cursor()
util.create_schema(c)
auth = util.authinfo(c)
(q, option) = get_option(c, q)
last_q = q.split(' ')[-1]
if q.startswith('_'): # option
process_option(c, q)
elif q.startswith('+'): # add bookmark
add_bookmark(c, q)
elif last_q.startswith('#') and (':' not in q): # tag expansion
pbsearch_tag(c, '', last_q[1:])
else:
pbsearch_sql(c, option, q)
util.closedb(conn)
if __name__ == '__main__':
try:
statsd = StatsClient(host='g.jmjeong.com',
port=8125,
prefix='jmjeong.alfred.bookmark')
with statsd.timer('main'):
statsd.incr('launch')
main()
except:
main()
from statsd import StatsClient
from datetime import datetime
from time import sleep
statsd_client = StatsClient(host='metrics')
'''
starting to include tags (remember: tags have to be strings)
'''
for x in range(100,1000,100):
print 'sleeping for {0} ms'.format(x)
with statsd_client.timer('sd_timer,tag1=foo,x={}'.format(x)):
sleep(float(x)/float(1000))
# Uptime
pipe.gauge('os.uptime', uptime.uptime())
# Host clock offset
try:
response = ntplib.NTPClient().request('pool.ntp.org')
except ntplib.NTPException:
pass
else:
pipe.gauge('os.ntp.offset', response.offset)
start = used = 0
metric = ns('watchtower', 'gathering')
while True:
start = int(time.time())
with statsd.timer(metric):
loadavg()
ram()
network()
os_status()
used = int(time.time()) - start
try:
time.sleep(10 - used) # sleep for the remainder of the interval
except IOError as exc:
print "IOError on time.sleep(10 - %r): %s" % (used, exc)
# Default sleep after an error
time.sleep(8)
conn = util.opendb()
c = conn.cursor()
util.create_schema(c)
auth = util.authinfo(c)
(q,option) = get_option(c,q)
last_q = q.split(' ')[-1]
if q.startswith('_'): # option
process_option(c,q)
elif q.startswith('+'): # add bookmark
add_bookmark(c,q)
elif last_q.startswith('#') and (':' not in q): # tag expansion
pbsearch_tag(c,'',last_q[1:])
else:
pbsearch_sql(c,option,q)
util.closedb(conn)
if __name__ == '__main__':
try:
statsd = StatsClient(host='g.jmjeong.com',
port=8125,
prefix='jmjeong.alfred.bookmark')
with statsd.timer('main'):
statsd.incr('launch');
main()
except:
main()
class Client(object):
def __init__(self, server, index='time_*', zero_fill=True, **kw):
self.server = server
self.session = requests.session()
self.es = urljoin(self.server, index + '/_search')
self.zero_fill = zero_fill
# statsd settings
statsd_host = kw.get('statsd.host', 'localhost')
statsd_port = int(kw.get('statsd.port', 8125))
statsd_prefix = kw.get('statsd.prefix', 'monolith.client')
self.statsd = StatsClient(host=statsd_host, port=statsd_port,
prefix=statsd_prefix)
def raw(self, query):
with self.statsd.timer('elasticsearch-query'):
res = self.session.get(self.es, data=json.dumps(query))
if res.status_code != 200:
raise ValueError(res.content)
# Get the JSON content.
res = res.json
if callable(res):
res = res()
self.statsd.incr('elasticsearch-call')
if not isinstance(res, dict):
raise ValueError(res)
if 'errors' in res:
raise ValueError(res['errors'][0]['description'])
return res
def __call__(self, field, start, end, interval=DAY, strict_range=False,
**terms):
if isinstance(interval, basestring):
interval = _str2interval[interval.encode()]
if isinstance(start, basestring):
start = datetime.datetime.strptime(start.encode(),
'%Y-%m-%d').toordinal()
start = datetime.date.fromordinal(start)
elif isinstance(start, datetime.datetime):
start = start.date()
if isinstance(end, basestring):
end = datetime.datetime.strptime(end.encode(),
'%Y-%m-%d').toordinal()
end = datetime.date.fromordinal(end)
elif isinstance(end, datetime.datetime):
end = end.date()
if interval == DAY:
drange = util.iterdays(start, end)
elif interval == WEEK:
drange = util.iterweeks(start, end)
elif interval == MONTH:
drange = util.itermonths(start, end)
else:
drange = util.iteryears(start, end)
# building the query
start_date_str = start.strftime('%Y-%m-%d')
end_date_str = end.strftime('%Y-%m-%d')
if isinstance(interval, int):
interval = _interval2str[interval]
# XXX we'll see later if we want to provide a
# nicer query interface
# we need a facet query
if strict_range:
greater = "gt"
lower = "lt"
else:
greater = "gte"
lower = "lte"
query = {
"query": {
"match_all": {},
},
"size": 0, # we aren't interested in the hits
"facets": {
"histo1": {
"date_histogram": {
"value_field": field,
"interval": interval,
"key_field": "date",
},
"facet_filter": {
"range": {
"date": {
greater: start_date_str,
lower: end_date_str,
}
}
}
}
}
}
if terms:
range_ = query['facets']['histo1']['facet_filter']['range']
query['facets']['histo1']['facet_filter'] = {
'and': ([{'term': {k: v}} for k, v in terms.items()] +
[{'range': range_}])}
res = self.raw(query)
counts = {}
for entry in res['facets']['histo1']['entries']:
time_ = entry['time'] / 1000.0
date_ = datetime.datetime.utcfromtimestamp(time_).date()
if 'total' in entry:
count = entry['total']
else:
count = entry['count']
counts[date_] = count
for date_ in drange:
if strict_range and date_ in (start, end):
continue
if date_ in counts:
yield {'count': counts[date_], 'date': date_}
elif self.zero_fill:
yield {'count': None, 'date': date_}
class Worker(multiprocessing.Process):
def __init__(self, worker_id, manager, redis_connection_params, sleep_time=0.1):
self.manager = manager
self.statsd_client = StatsClient(host=settings.STATSD_HOST, port=settings.STATSD_PORT,
prefix=settings.STATSD_PREFIX)
self.redis_connection_params = {k: v for k, v in redis_connection_params.iteritems()
if k in ('host', 'db', 'password', 'port')}
self.worker_id = None
self.continue_working = True
self.sleep_time = sleep_time
self.child_pid = None
self.current_job_id = None
self.status = {
'jobs_count': 0,
'cancelled_jobs_count': 0,
'done_jobs_count': 0,
'updated_at': time.time(),
'started_at': time.time()
}
super(Worker, self).__init__(name="Worker")
def set_title(self, title=None):
base_title = "redash worker:%s" % self.worker_id
if title:
full_title = "%s - %s" % (base_title, title)
else:
full_title = base_title
setproctitle.setproctitle(full_title)
def run(self):
self.worker_id = os.getpid()
self.status['id'] = self.worker_id
self.name = "Worker:%d" % self.worker_id
self.manager.redis_connection.sadd('workers', self._key)
self._save_status()
self.set_title()
logging.info("[%s] started.", self.name)
signal.signal(signal.SIGINT, self._stop)
signal.signal(signal.SIGTERM, self._stop)
self._wait_for_jobs()
def _stop(self, signum, frame):
self.continue_working = False
if self.current_job_id:
job = Job.load(self.manager.redis_connection, self.current_job_id)
if job:
job.cancel()
def _wait_for_jobs(self):
while self.continue_working:
job_id = self.manager.queue.pop()
if job_id:
self._update_status('jobs_count')
logging.info("[%s] Processing %s", self.name, job_id)
self._fork_and_process(job_id)
if self.child_pid == 0:
return
else:
time.sleep(self.sleep_time)
def _update_status(self, counter):
self.status['updated_at'] = time.time()
self.status[counter] += 1
self._save_status()
@property
def _key(self):
return 'worker:%s' % self.worker_id
def _save_status(self):
self.manager.redis_connection.hmset(self._key, self.status)
def _fork_and_process(self, job_id):
self.current_job_id = job_id
self.child_pid = os.fork()
if self.child_pid == 0:
self.set_title("processing %s" % job_id)
self._process(job_id)
else:
logging.info("[%s] Waiting for pid: %d", self.name, self.child_pid)
try:
_, status = os.waitpid(self.child_pid, 0)
except OSError:
logging.info("[%s] OSError while waiting for child to finish", self.name)
# setting status to >0, so the job cleanup is triggered
status = 1
self._update_status('done_jobs_count')
job = Job.load(self.manager.redis_connection, job_id)
if status > 0 and not job.is_finished():
self._update_status('cancelled_jobs_count')
logging.info("[%s] process interrupted and job %s hasn't finished; registering interruption in job",
self.name, job_id)
job.done(None, "Interrupted/Cancelled while running.")
job.expire(settings.JOB_EXPIRY_TIME)
logging.info("[%s] Finished Processing %s (pid: %d status: %d)",
self.name, job_id, self.child_pid, status)
self.child_pid = None
self.current_job_id = None
def _process(self, job_id):
redis_connection = redis.StrictRedis(**self.redis_connection_params)
job = Job.load(redis_connection, job_id)
if job.is_finished():
logging.warning("[%s][%s] tried to process finished job.", self.name, job)
return
pid = os.getpid()
job.processing(pid)
logging.info("[%s][%s] running query...", self.name, job.id)
start_time = time.time()
self.set_title("running query %s" % job_id)
logging.info("[%s][%s] Loading query runner (%s, %s)...", self.name, job.id,
job.data_source_name, job.data_source_type)
query_runner = get_query_runner(job.data_source_type, job.data_source_options)
if getattr(query_runner, 'annotate_query', True):
annotated_query = "/* Pid: %s, Job Id: %s, Query hash: %s, Priority: %s */ %s" % \
(pid, job.id, job.query_hash, job.priority, job.query)
else:
annotated_query = job.query
# TODO: here's the part that needs to be forked, not all of the worker process...
with self.statsd_client.timer('worker_{}.query_runner.{}.{}.run_time'.format(self.worker_id,
job.data_source_type,
job.data_source_name)):
data, error = query_runner(annotated_query)
run_time = time.time() - start_time
logging.info("[%s][%s] query finished... data length=%s, error=%s",
self.name, job.id, data and len(data), error)
# TODO: it is possible that storing the data will fail, and we will need to retry
# while we already marked the job as done
query_result_id = None
if not error:
self.set_title("storing results %s" % job_id)
query_result_id = self.manager.store_query_result(job.data_source_id,
job.query, data, run_time,
datetime.datetime.utcnow())
self.set_title("marking job as done %s" % job_id)
job.done(query_result_id, error)
class StatsdMonitor(object):
def __init__(self, broker, interval=1):
# self.interval = interval
self.state = app.events.State()
self.statsd_conn = StatsClient(host='localhost', port=8125)
self.broker_conn = BrokerConnection(broker)
self.timers_list = []
# monitor the task and status of worker with functions
def run_loop(self):
while True:
try:
with self.broker_conn as conn:
recv = EventReceiver(conn,
handlers={
'task-sent':
self.on_task_sent,
'task-failed':
self.on_task_failed,
'task-retried':
self.on_task_retried,
'task-started':
self.on_task_started,
'task-succeeded':
self.on_task_succeeded,
'task-received':
self.on_task_received,
'task-rejected':
self.on_task_rejected,
'task-revoked':
self.on_task_revoked,
'worker-online':
self.on_worker_online,
'worker-heartbeat':
self.on_worker_heartbeat,
'worker-offline':
self.on_worker_offline,
})
recv.capture(limit=None, timeout=None, wakeup=True)
except (KeyboardInterrupt, SystemExit):
raise
except Exception:
raise
# time.sleep(self.interval)
# all about the tasks
def on_task_sent(self, event): # TODO
self.state.event(event)
task = self.state.tasks.get(event['uuid'])
self.statsd_conn.incr('tasks.sent')
def on_task_received(self, event): # TODO
self.state.event(event)
task = self.state.tasks.get(event['uuid'])
self.statsd_conn.incr('tasks.received')
def on_task_started(self, event):
self.state.event(event)
task = self.state.tasks.get(event['uuid'])
logger.info('Task {}[{}] started'.format(task.name, task.uuid))
self.statsd_conn.incr('tasks.started')
mark = 'task.{}.recorder'.format(task.uuid)
self.timer_start(mark)
def on_task_succeeded(self, event):
self.state.event(event)
task = self.state.tasks.get(event['uuid'])
logger.info('Task {}[{}] succeeded'.format(task.name, task.uuid))
self.statsd_conn.incr('tasks.succeeded')
mark = 'task.{}.recorder'.format(task.uuid)
self.timer_stop(mark)
def on_task_failed(self, event): # TODO
self.state.event(event)
task = self.state.tasks.get(event['uuid'])
logger.warning('Task {}[{}] failed'.format(task.name, task.uuid))
self.statsd_conn.incr('tasks.failed')
def on_task_retried(self, event): # TODO
self.state.event(event)
task = self.state.tasks.get(event['uuid'])
logger.warning('Task {}[{}] retried'.format(task.name, task.uuid))
self.statsd_conn.incr('tasks.retried')
def on_task_rejected(self, event): # TODO
self.state.event(event)
task = self.state.tasks.get(event['uuid'])
def on_task_revoked(self, event): # TODO
self.state.event(event)
task = self.state.tasks.get(event['uuid'])
# all about the status of the workers
def on_worker_online(self, event): # TODO
self.state.event(event)
worker = self.state.workers.get(event['hostname'])
mark = 'worker.{}.recorder'.format(worker.hostname)
self.timer_start(mark)
def on_worker_heartbeat(self, event):
self.state.event(event)
worker = self.state.workers.get(event['hostname'])
key_pro = 'worker.{}.processed'.format(worker.hostname)
key_act = 'worker.{}.active'.format(worker.hostname)
if worker.processed is None: worker.processed = 0
if worker.active is None: worker.active = 0
self.statsd_conn.gauge(key_pro, worker.processed)
self.statsd_conn.gauge(key_act, worker.active)
def on_worker_offline(self, event): # TODO
self.state.event(event)
worker = self.state.workers.get(event['hostname'])
mark = 'worker.{}.recorder'.format(worker.hostname)
self.timer_stop(mark)
# statsd timer record start
def timer_start(self, mark):
timer = self.statsd_conn.timer(mark)
timer.start()
self.timers_list.append(timer)
# statsd timer record stop
def timer_stop(self, mark):
for timer in self.timers_list:
if timer.stat == mark:
timer.stop()
self.timers_list.remove(timer)
from statsd import StatsClient
from datetime import datetime
from time import sleep
statsd_client = StatsClient(host='metrics')
for x in range(100,1000,100): # start of loop
print 'sleeping for {0} ms'.format(x) # print sleep time
with statsd_client.timer('sd_timer'): # begin statsd timer
sleep(float(x)/float(1000)) # sleep X milliseconds
with open(CONFIG_FILE, 'r') as f:
data = json.load(f)
REDIS_SERVER_HOST = data['redis']['redisServerHost']
REDIS_SERVER_PORT = int(data['redis']['redisServerPort'])
REDIS_NEWS_EXPIRE_IN_SECONDS = int(data['redis']['newsExpireInSeconds'])
NEWS_LIMIT = int(data['news']['newsLimit'])
NEWS_PER_PAGE_SIZE = int(data['news']['newsPageSize'])
NEWS_DB_COLLECTION = data['mongoDb']['newsMongoDbCollection']
CLICKS_DB_COLLECTION = data['mongoDb']['clicksMongoDbCollection']
LOG_CLICKS_TASK_QUEUE_URL = data['queue']['logClicksTaskQueueUrl']
LOG_CLICKS_TASK_QUEUE_NAME = data['queue']['logClicksTaskQueueName']
redis_client = redis.StrictRedis(REDIS_SERVER_HOST, REDIS_SERVER_PORT)
CloudAMQPClient = CloudAMQPClient(LOG_CLICKS_TASK_QUEUE_URL, LOG_CLICKS_TASK_QUEUE_NAME)
statsd = StatsClient()
news_timer = statsd.timer('news')
def isclose(a, b, rel_tol=1e-09, abs_tol=0.0):
return abs(a-b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol)
def getNewssummaryForuser(userId, pageNum):
news_timer.start()
print "get news for user: %s @ page %s" % (userId, pageNum)
statsd.incr("user.news")
pageNum = int(pageNum)
startIndex = (pageNum - 1) * NEWS_PER_PAGE_SIZE
endIndex = pageNum * NEWS_PER_PAGE_SIZE
sliced_news = []
#get preference for user
pref_dict = news_recommendation_service_client.getPreferenceForuser(userId)
from statsd import StatsClient
statsd = StatsClient()
foo_timer = statsd.timer('foo')
foo_timer.start()
# Do something fun.
foo_timer.stop()
