def _send(self):
"""
Send data to statsd. Fire and forget. Cross fingers and it'll arrive.
"""
if not statsd:
return
for metric in self.metrics:
# Split the path into a prefix and a name
# to work with the statsd module's view of the world.
# It will get re-joined by the python-statsd module.
(prefix, name) = metric.path.rsplit(".", 1)
logging.debug("Sending %s %s|g", name, metric.value)
if metric.metric_type == 'GAUGE':
statsd.Gauge(prefix, self.connection).send(name, metric.value)
else:
# To send a counter, we need to just send the delta
# but without any time delta changes
value = metric.raw_value
if metric.path in self.old_values:
value = value - self.old_values[metric.path]
self.old_values[metric.path] = metric.raw_value
statsd.Counter(prefix, self.connection).increment(name, value)
self.metrics = []
def dec_counter(self, key, amount=1):
""" Decrement metric
"""
check_key(key)
assert isinstance(amount, Number)
self._counters[key] = self._counters.get(key, statsd.Counter(key))
self._counters[key] -= amount
def inc_counter(self, key, amount=1):
""" Increment counter
"""
check_key(key)
assert isinstance(amount, Number)
self._counters[key] = self._counters.get(key, statsd.Counter(key))
self._counters[key] += amount
def Counter(name):
return statsd.Counter("%s.%s" % (get_prefix(), name))
from metrics import Counter
counter = Counter(__name__)
counter += 1
def handle_message(msg):
if msg is None:
log_client.logger.info('message is broken')
#print 'message is broken'
return
counter = statsd.Counter(msg)
counter += 1
def post_event_to_webhook_ee(self: Task, event: Dict[str, Any], team_id: int, site_url: str) -> None:
if not site_url:
site_url = settings.SITE_URL
timer = statsd.Timer("%s_posthog_cloud" % (settings.STATSD_PREFIX,))
timer.start()
team = Team.objects.select_related("organization").get(pk=team_id)
elements_list = chain_to_elements(event.get("elements_chain", ""))
ephemeral_postgres_event = Event.objects.create(
event=event["event"],
distinct_id=event["distinct_id"],
properties=event["properties"],
team=team,
site_url=site_url,
**({"timestamp": event["timestamp"]} if event["timestamp"] else {}),
**({"elements": elements_list})
)
try:
is_zapier_available = team.organization.is_feature_available("zapier")
actionFilters = {"team_id": team_id}
if not is_zapier_available:
if not team.slack_incoming_webhook:
return # Exit this task if neither Zapier nor webhook URL are available
else:
actionFilters["post_to_slack"] = True # We only need to fire for actions that are posted to webhook URL
for action in cast(Sequence[Action], Action.objects.filter(**actionFilters).all()):
qs = Event.objects.filter(pk=ephemeral_postgres_event.pk).query_db_by_action(action)
if not qs:
continue
# REST hooks
if is_zapier_available:
action.on_perform(ephemeral_postgres_event)
# webhooks
if team.slack_incoming_webhook and action.post_to_slack:
message_text, message_markdown = get_formatted_message(action, ephemeral_postgres_event, site_url)
if determine_webhook_type(team) == "slack":
message = {
"text": message_text,
"blocks": [{"type": "section", "text": {"type": "mrkdwn", "text": message_markdown}}],
}
else:
message = {
"text": message_markdown,
}
statsd.Counter("%s_posthog_cloud_hooks_web_fired" % (settings.STATSD_PREFIX)).increment()
requests.post(team.slack_incoming_webhook, verify=False, json=message)
except:
raise
finally:
timer.stop("hooks_processed_for_event")
ephemeral_postgres_event.delete()
def inc_counter(self, key, amount=1):
"""
Increment counter
"""
check_key(key)
assert isinstance(amount, Number)
key = get_full_key_name(key)
counter = statsd.Counter(key)
counter.increment(delta=amount)
def dec_counter(self, key, amount=1):
"""
Decrement metric
"""
check_key(key)
assert isinstance(amount, Number)
key = get_full_key_name(key)
counter = statsd.Counter(key)
counter.decrement(delta=amount)
def statsd_counter(name, value):
try:
if not isinstance(value, (float, int)):
value = float(value)
if not ig_conf.STATSD_HOST:
return
else:
counter = statsd.Counter(ig_conf.STATSD_PREFIX)
counter.increment(name, value)
except Exception as e:
logger.exception(e)
def increase_counter(name, value=1):
"""
Sends one-time counter to all backends configured for this setup
"""
try:
if not isinstance(value, (float, int)):
value = float(value)
if ig_conf.PRINT_CONSOLE_METRICS:
print(f"\t{name}:\t{value}")
if ig_conf.STATSD_HOST:
counter = statsd.Counter(ig_conf.STATSD_PREFIX)
counter.increment(name, value)
if ig_conf.SEND_CLOUDWATCH_METRICS:
_send_cloudwatch_metric(name, value, unit="Count")
except Exception as e:
logger.exception(e)
def find_and_fire_hook(
event_name: str,
instance: models.Model,
user_override: Team,
payload_override: Optional[dict] = None,
):
if not user_override.organization.is_feature_available("zapier"):
return
hooks = Hook.objects.filter(event=event_name, team=user_override)
if event_name == "action_performed":
# action_performed is a resource_id-filterable hook
hooks = hooks.filter(models.Q(resource_id=instance.pk))
for hook in hooks:
statsd.Counter("%s_posthog_cloud_hooks_rest_fired" %
(settings.STATSD_PREFIX, )).increment()
hook.deliver_hook(instance, payload_override)
def send_messages(self):
'''Main processing for sending messages.'''
try:
conn = mstatsd.Connection(host=self.host, port=self.port)
self.client = mstatsd.Client(name='statsd-generator', connection=conn)
for index in range(1, self.num_of_iterations + 1):
print("Starting iteration " + str(index) +
" of " + str(self.num_of_iterations))
counter = self.client.get_counter('teraflops')
counter.increment(5)
gauge = self.client.get_gauge()
gauge.send('num_of_teraflops',
random.uniform(1.0, 10.0),
dimensions={'origin': 'dev',
'environment': 'test'})
histogram = self.client.get_histogram('hist')
histogram.send('file.upload.size',
random.randrange(1, 100),
dimensions={'version': '1.0'})
set = self.client.get_set('hist')
set.send('load_time',
random.randrange(1, 100),
dimensions={'page_name': 'mypage.html'})
timer = self.client.get_timer('timer')
@timer.timed('config_db_time',
dimensions={'db_name': 'mydb'})
def time_db():
time.sleep(0.2)
time_db()
with timer.time('time_block'):
time.sleep(0.3)
# Send some regular statsd messages
counter = statsd.Counter('statsd_counter')
counter += 1
gauge = statsd.Gauge('statsd_gauge')
gauge.send('cpu_percent',
random.uniform(1.0, 100.0))
print("Completed iteration " + str(index) +
". Sleeping for " + str(self.delay) + " seconds...")
time.sleep(self.delay)
except Exception:
print ("Error sending statsd messages...")
raise
def hello():
counter = statsd.Counter("Homepage hits")
counter += 1
return "Hello World!"
def prepare_statsd(parameters):
r"""Sends data to statsd
Sends a value to statsd.
host
defaults to ``127.0.0.1``
port
defaults to ``8125``
sample_rate
defaults to ``1.0``
type
Accepted values are ``counter``, ``gauge`` and ``timer``, defaults to
``counter``
value
The value to send. Defaults to ``1.0``
multiplier
The amount to multiply the value by. Defaults to ``1.0``
delta
boolean, only used for gauge, whether to send differential values or
absolute values. Defaults to ``False``
prefix
the prefix for the stat name backreferences not allowed
name
the name for the stat, backreferences allowed (required)
Example:
.. code:: yaml
match: Duration: (\d+.\d+)s
statsd:
type: timer
value: {1}
prefix: appserver.request
name: duration
statsd:
prefix: appserver.request
name: count
"""
import statsd # noqa
statsd_connection = statsd.Connection(
host=parameters.get('host', '127.0.0.1'),
port=int(parameters.get('port', 8125)),
sample_rate=float(parameters.get('sample_rate', 1.0)),
)
meter_type = parameters.get('type', 'counter')
name_template = logshipper.context.prepare_template(parameters['name'])
val_template = logshipper.context.prepare_template(
parameters.get('value', 1))
multiplier = float(parameters.get('multiplier', 1.0))
if meter_type == 'counter':
statsd_client = statsd.Counter(parameters.get('prefix'),
statsd_connection)
delta = True
elif meter_type == 'gauge':
statsd_client = statsd.Gauge(parameters.get('prefix'),
statsd_connection)
delta_str = str(parameters.get("delta", False)).lower()
delta = delta_str in filters.TRUTH_VALUES
elif meter_type == 'timer':
statsd_client = statsd.Timer(parameters.get('prefix'),
statsd_connection)
delta = False
else:
raise ValueError("Unknown meter type, should be one of counter, "
"gauge or timer") # pragma: nocover
def handle_statsd(message, context):
name = name_template.interpolate(context)
value = val_template.interpolate(context)
if delta:
statsd_client.increment(name, float(value) * multiplier)
else:
statsd_client.send(name, float(value) * multiplier)
return handle_statsd
def incr(self, key):
counter = statsd.Counter(key)
counter += 1
from __future__ import with_statement
import mock
import statsd
with mock.patch('statsd.Client') as mock_client:
instance = mock_client.return_value
instance._send.return_value = 1
counter = statsd.Counter('testing')
counter.increment('')
mock_client._send.assert_called_with(mock.ANY, {'testing': '1|c'})
counter.increment('', 2)
mock_client._send.assert_called_with(mock.ANY, {'testing': '2|c'})
counter.decrement('')
mock_client._send.assert_called_with(mock.ANY, {'testing': '-1|c'})
counter.decrement('', 2)
mock_client._send.assert_called_with(mock.ANY, {'testing': '-2|c'})
def create_event(
event_uuid: uuid.UUID,
event: str,
team: Team,
distinct_id: str,
timestamp: Optional[Union[timezone.datetime, str]] = None,
properties: Optional[Dict] = {},
elements: Optional[List[Element]] = None,
site_url: Optional[str] = None,
) -> str:
if not timestamp:
timestamp = timezone.now()
assert timestamp is not None
# clickhouse specific formatting
if isinstance(timestamp, str):
timestamp = isoparse(timestamp)
else:
timestamp = timestamp.astimezone(pytz.utc)
elements_chain = ""
if elements and len(elements) > 0:
elements_chain = elements_to_string(elements=elements)
pb_event = events_pb2.Event()
pb_event.uuid = str(event_uuid)
pb_event.event = event
pb_event.properties = json.dumps(properties)
pb_event.timestamp = timestamp.strftime("%Y-%m-%d %H:%M:%S.%f")
pb_event.team_id = team.pk
pb_event.distinct_id = str(distinct_id)
pb_event.elements_chain = elements_chain
pb_event.created_at = timestamp.strftime("%Y-%m-%d %H:%M:%S.%f")
p = ClickhouseProducer()
p.produce_proto(sql=INSERT_EVENT_SQL, topic=KAFKA_EVENTS, data=pb_event)
if team.slack_incoming_webhook or (
team.organization.is_feature_available("zapier")
and Hook.objects.filter(event="action_performed", team=team).exists()
):
try:
statsd.Counter("%s_posthog_cloud_hooks_send_task" % (settings.STATSD_PREFIX,)).increment()
celery.current_app.send_task(
"ee.tasks.webhooks_ee.post_event_to_webhook_ee",
(
{
"event": event,
"properties": properties,
"distinct_id": distinct_id,
"timestamp": timestamp,
"elements_chain": elements_chain,
},
team.pk,
site_url,
),
)
except:
capture_exception()
return str(event_uuid)
while True:
timer = statsd.Timer(metrics_prefix)
timer.start()
fbx.update()
timer.stop("dataAcquisitionTime")
gauge = statsd.Gauge(metrics_prefix)
gauge.send("connection.debit.down",
fbx.status["adsl"]["synchro_speed"]["down"])
gauge.send("connection.debit.up",
fbx.status["adsl"]["synchro_speed"]["up"])
gauge.send("network.WAN.down",
fbx.status["network"]["interfaces"]["WAN"]["down"])
gauge.send("network.WAN.up",
fbx.status["network"]["interfaces"]["WAN"]["up"])
gauge.send("network.ethernet.down",
fbx.status["network"]["interfaces"]["ethernet"]["down"])
gauge.send("network.ethernet.up",
fbx.status["network"]["interfaces"]["ethernet"]["up"])
gauge.send("network.switch.down",
fbx.status["network"]["interfaces"]["switch"]["down"])
gauge.send("network.switch.up",
fbx.status["network"]["interfaces"]["switch"]["up"])
c = statsd.Counter(metrics_prefix)
c.increment("telephone.sonnerie",
1 if fbx.status["telephone"]["ringing"] else 0)
c.increment("telephone.en_ligne",
1 if fbx.status["telephone"]["online"] else 0)
time.sleep(30)
def on_feed_remove(self, feed_class, activities_count):
counter = statsd.Counter('%s.%s.deletes' %
(self.prefix, feed_class.__name__))
counter += activities_count
def on_activity_published(self):
counter = statsd.Counter('%s.activities.published' % self.prefix)
counter += 1
def on_fanout(self, feed_class, operation, activities_count=1):
metric = (self.prefix, feed_class.__name__, operation.__name__)
counter = statsd.Counter('%s.%s.fanout.%s' % metric)
counter += activities_count
def setUp(self):
self.counter = statsd.Counter('testing')
def on_activity_removed(self):
counter = statsd.Counter('%s.activities.removed' % self.prefix)
counter += 1
def Counter(name, suffix=None):
if suffix:
name = append_suffix(name, suffix)
return statsd.Counter("%s.%s" % (get_prefix(), name))
def statsd_metric_task(slug, num=1, **kwargs):
conn = get_statsd_conn()
counter = statsd.Counter(slug, connection=conn)
counter += num
def setUp(self):
self.con = statsd.Connection(host=HOST, port=PORT, disabled=False)
self.gauge = statsd.Gauge("core_prod", self.con)
self.counter = statsd.Counter("core_prod", self.con)
def get_event(request):
timer = statsd.Timer("%s_posthog_cloud" % (settings.STATSD_PREFIX,))
timer.start()
now = timezone.now()
try:
data = load_data_from_request(request)
except RequestParsingError as error:
capture_exception(error) # We still capture this on Sentry to identify actual potential bugs
return cors_response(
request, generate_exception_response(f"Malformed request data: {error}", code="invalid_payload"),
)
if not data:
return cors_response(
request,
generate_exception_response(
"No data found. Make sure to use a POST request when sending the payload in the body of the request.",
code="no_data",
),
)
sent_at = _get_sent_at(data, request)
token = _get_token(data, request)
if not token:
return cors_response(
request,
generate_exception_response(
"API key not provided. You can find your project API key in PostHog project settings.",
type="authentication_error",
code="missing_api_key",
status_code=status.HTTP_401_UNAUTHORIZED,
),
)
team = Team.objects.get_team_from_token(token)
if team is None:
try:
project_id = _get_project_id(data, request)
except ValueError:
return cors_response(
request, generate_exception_response("Invalid Project ID.", code="invalid_project", attr="project_id"),
)
if not project_id:
return cors_response(
request,
generate_exception_response(
"Project API key invalid. You can find your project API key in PostHog project settings.",
type="authentication_error",
code="invalid_api_key",
status_code=status.HTTP_401_UNAUTHORIZED,
),
)
user = User.objects.get_from_personal_api_key(token)
if user is None:
return cors_response(
request,
generate_exception_response(
"Invalid Personal API key.",
type="authentication_error",
code="invalid_personal_api_key",
status_code=status.HTTP_401_UNAUTHORIZED,
),
)
team = user.teams.get(id=project_id)
if isinstance(data, dict):
if data.get("batch"): # posthog-python and posthog-ruby
data = data["batch"]
assert data is not None
elif "engage" in request.path_info: # JS identify call
data["event"] = "$identify" # make sure it has an event name
if isinstance(data, list):
events = data
else:
events = [data]
try:
events = preprocess_session_recording_events(events)
except ValueError as e:
return cors_response(request, generate_exception_response(f"Invalid payload: {e}", code="invalid_payload"))
for event in events:
try:
distinct_id = _get_distinct_id(event)
except KeyError:
return cors_response(
request,
generate_exception_response(
"You need to set user distinct ID field `distinct_id`.", code="required", attr="distinct_id"
),
)
if not event.get("event"):
return cors_response(
request,
generate_exception_response(
"You need to set user event name, field `event`.", code="required", attr="event"
),
)
if not event.get("properties"):
event["properties"] = {}
_ensure_web_feature_flags_in_properties(event, team, distinct_id)
event_uuid = UUIDT()
ip = None if team.anonymize_ips else get_ip_address(request)
if is_ee_enabled():
log_topics = [KAFKA_EVENTS_WAL]
if settings.PLUGIN_SERVER_INGESTION:
log_topics.append(KAFKA_EVENTS_PLUGIN_INGESTION)
statsd.Counter("%s_posthog_cloud_plugin_server_ingestion" % (settings.STATSD_PREFIX,)).increment()
log_event(
distinct_id=distinct_id,
ip=ip,
site_url=request.build_absolute_uri("/")[:-1],
data=event,
team_id=team.id,
now=now,
sent_at=sent_at,
event_uuid=event_uuid,
topics=log_topics,
)
# must done after logging because process_event_ee modifies the event, e.g. by removing $elements
if not settings.PLUGIN_SERVER_INGESTION:
process_event_ee(
distinct_id=distinct_id,
ip=ip,
site_url=request.build_absolute_uri("/")[:-1],
data=event,
team_id=team.id,
now=now,
sent_at=sent_at,
event_uuid=event_uuid,
)
else:
task_name = "posthog.tasks.process_event.process_event_with_plugins"
celery_queue = settings.PLUGINS_CELERY_QUEUE
celery_app.send_task(
name=task_name,
queue=celery_queue,
args=[distinct_id, ip, request.build_absolute_uri("/")[:-1], event, team.id, now.isoformat(), sent_at,],
)
timer.stop("event_endpoint")
return cors_response(request, JsonResponse({"status": 1}))
def put(self, metric, value, units):
with connectionLock:
global connection
statsd.Counter(self.namespace, connection).increment(metric, value)
def get_event(request):
timer = statsd.Timer("%s_posthog_cloud" % (settings.STATSD_PREFIX, ))
timer.start()
now = timezone.now()
try:
data_from_request = load_data_from_request(request)
data = data_from_request["data"]
except TypeError:
return cors_response(
request,
JsonResponse(
{
"code":
"validation",
"message":
"Malformed request data. Make sure you're sending valid JSON.",
},
status=400,
),
)
if not data:
return cors_response(
request,
JsonResponse(
{
"code":
"validation",
"message":
"No data found. Make sure to use a POST request when sending the payload in the body of the request.",
},
status=400,
),
)
sent_at = _get_sent_at(data, request)
token = _get_token(data, request)
if not token:
return cors_response(
request,
JsonResponse(
{
"code":
"validation",
"message":
"API key not provided. You can find your project API key in PostHog project settings.",
},
status=401,
),
)
team = Team.objects.get_team_from_token(token)
if team is None:
try:
project_id = _get_project_id(data, request)
except:
return cors_response(
request,
JsonResponse(
{
"code": "validation",
"message": "Invalid project ID.",
},
status=400,
),
)
if not project_id:
return cors_response(
request,
JsonResponse(
{
"code":
"validation",
"message":
"Project API key invalid. You can find your project API key in PostHog project settings.",
},
status=401,
),
)
user = User.objects.get_from_personal_api_key(token)
if user is None:
return cors_response(
request,
JsonResponse(
{
"code": "validation",
"message": "Personal API key invalid.",
},
status=401,
),
)
team = user.teams.get(id=project_id)
if isinstance(data, dict):
if data.get("batch"): # posthog-python and posthog-ruby
data = data["batch"]
assert data is not None
elif "engage" in request.path_info: # JS identify call
data["event"] = "$identify" # make sure it has an event name
if isinstance(data, list):
events = data
else:
events = [data]
for event in events:
try:
distinct_id = _get_distinct_id(event)
except KeyError:
return cors_response(
request,
JsonResponse(
{
"code": "validation",
"message":
"You need to set user distinct ID field `distinct_id`.",
"item": event,
},
status=400,
),
)
if not event.get("event"):
return cors_response(
request,
JsonResponse(
{
"code": "validation",
"message": "You need to set event name field `event`.",
"item": event,
},
status=400,
),
)
if not event.get("properties"):
event["properties"] = {}
_ensure_web_feature_flags_in_properties(event, team, distinct_id)
event_uuid = UUIDT()
if is_ee_enabled():
log_topics = [KAFKA_EVENTS_WAL]
if settings.PLUGIN_SERVER_INGESTION:
log_topics.append(KAFKA_EVENTS_PLUGIN_INGESTION)
statsd.Counter("%s_posthog_cloud_plugin_server_ingestion" %
(settings.STATSD_PREFIX, )).increment()
log_event(
distinct_id=distinct_id,
ip=get_ip_address(request),
site_url=request.build_absolute_uri("/")[:-1],
data=event,
team_id=team.id,
now=now,
sent_at=sent_at,
event_uuid=event_uuid,
topics=log_topics,
)
# must done after logging because process_event_ee modifies the event, e.g. by removing $elements
if not settings.PLUGIN_SERVER_INGESTION:
process_event_ee(
distinct_id=distinct_id,
ip=get_ip_address(request),
site_url=request.build_absolute_uri("/")[:-1],
data=event,
team_id=team.id,
now=now,
sent_at=sent_at,
event_uuid=event_uuid,
)
else:
task_name = "posthog.tasks.process_event.process_event"
if settings.PLUGIN_SERVER_INGESTION or team.plugins_opt_in:
task_name += "_with_plugins"
celery_queue = settings.PLUGINS_CELERY_QUEUE
else:
celery_queue = settings.CELERY_DEFAULT_QUEUE
celery_app.send_task(
name=task_name,
queue=celery_queue,
args=[
distinct_id,
get_ip_address(request),
request.build_absolute_uri("/")[:-1],
event,
team.id,
now.isoformat(),
sent_at,
],
)
timer.stop("event_endpoint")
return cors_response(request, JsonResponse({"status": 1}))
def count(self, metric):
with connectionLock:
global connection
statsd.Counter(self.namespace, connection).increment(metric, 1)