def test_request_retry(self):
# we should retry on general errors
consumer = Consumer(None, 'testsecret')
self._test_request_retry(consumer, Exception('generic exception'), 2)
# we should retry on server errors
consumer = Consumer(None, 'testsecret')
self._test_request_retry(
consumer, APIError(500, 'code', 'Internal Server Error'), 2)
# we should retry on HTTP 429 errors
consumer = Consumer(None, 'testsecret')
self._test_request_retry(consumer,
APIError(429, 'code', 'Too Many Requests'), 2)
# we should NOT retry on other client errors
consumer = Consumer(None, 'testsecret')
api_error = APIError(400, 'code', 'Client Errors')
try:
self._test_request_retry(consumer, api_error, 1)
except APIError:
pass
else:
self.fail('request() should not retry on client errors')
# test for number of exceptions raise > retries value
consumer = Consumer(None, 'testsecret', retries=3)
self._test_request_retry(
consumer, APIError(500, 'code', 'Internal Server Error'), 3)
def __init__(self, write_key=None, host=None, debug=False, max_queue_size=10000,
send=True, on_error=None, upload_size=100, upload_interval=0.5,
gzip=False, max_retries=10):
require('write_key', write_key, string_types)
self.queue = queue.Queue(max_queue_size)
self.consumer = Consumer(self.queue, write_key, host=host, on_error=on_error,
upload_size=upload_size, upload_interval=upload_interval,
gzip=gzip, retries=max_retries)
self.write_key = write_key
self.on_error = on_error
self.debug = debug
self.send = send
if debug:
self.log.setLevel(logging.DEBUG)
# if we've disabled sending, just don't start the consumer
if send:
# On program exit, allow the consumer thread to exit cleanly.
# This prevents exceptions and a messy shutdown when the interpreter is
# destroyed before the daemon thread finishes execution. However, it
# is *not* the same as flushing the queue! To guarantee all messages
# have been delivered, you'll still need to call flush().
atexit.register(self.join)
self.consumer.start()
class Client(_Client):
def __init__(self,
write_key=None,
debug=False,
max_queue_size=10000,
send=True,
on_error=None):
# We need a different queue type to have a client that works properly across separate processes.
# This client is meant to be shared and used within celery environment (although it should work otherwise).
# This will hopefully fix: https://github.com/segmentio/analytics-python/issues/51 although it is uncertain
# what exactly is causing it.
require('write_key', write_key, str)
self.queue = JoinableQueue(max_queue_size)
self.consumer = Consumer(self.queue, write_key, on_error=on_error)
self.write_key = write_key
self.on_error = on_error
self.debug = debug
self.send = send
if debug:
self.log.setLevel(logging.DEBUG)
# if we've disabled sending, just don't start the consumer
if send:
self.consumer.start()
def test_max_batch_size(self):
q = Queue()
consumer = Consumer(q,
'testsecret',
upload_size=100000,
upload_interval=3)
track = {'type': 'track', 'event': 'python event', 'userId': 'userId'}
msg_size = len(json.dumps(track).encode())
# number of messages in a maximum-size batch
n_msgs = int(475000 / msg_size)
def mock_post_fn(_, data, **kwargs):
res = mock.Mock()
res.status_code = 200
self.assertTrue(
len(data.encode()) < 500000,
'batch size (%d) exceeds 500KB limit' % len(data.encode()))
return res
with mock.patch('analytics.request._session.post',
side_effect=mock_post_fn) as mock_post:
consumer.start()
for _ in range(0, n_msgs + 2):
q.put(track)
q.join()
self.assertEqual(mock_post.call_count, 2)
def test_upload(self):
q = Queue()
consumer = Consumer(q, 'testsecret')
track = {'type': 'track', 'event': 'python event', 'userId': 'userId'}
q.put(track)
success = consumer.upload()
self.assertTrue(success)
def test_dropping_oversize_msg(self):
q = Queue()
consumer = Consumer(q, '')
oversize_msg = {'m': 'x' * MAX_MSG_SIZE}
q.put(oversize_msg)
next = consumer.next()
self.assertEqual(next, [])
self.assertTrue(q.empty())
def test_next_limit(self):
q = Queue()
flush_at = 50
consumer = Consumer(q, '', flush_at)
for i in range(10000):
q.put(i)
next = consumer.next()
self.assertEqual(next, list(range(flush_at)))
def test_dropping_oversize_msg(self):
q = Queue()
consumer = Consumer(q, '')
oversize_msg = {'m': 'x' * MAX_MSG_SIZE}
q.put(oversize_msg)
next = consumer.next()
self.assertEqual(next, [])
self.assertTrue(q.empty())
def test_request(self):
consumer = Consumer(None, 'testsecret')
track = {
'type': 'track',
'event': 'python event',
'userId': 'userId'
}
consumer.request([track])
def test_next_limit(self):
q = Queue()
upload_size = 50
consumer = Consumer(q, '', upload_size)
for i in range(10000):
q.put(i)
next = consumer.next()
self.assertEqual(next, list(range(upload_size)))
def test_request(cls):
consumer = Consumer(None, 'testsecret')
track = {
'type': 'track',
'event': 'python event',
'userId': 'userId'
}
consumer.request([track])
def test_next_limit(self):
q = Queue()
upload_size = 50
consumer = Consumer(q, '', upload_size)
for i in range(10000):
q.put(i)
next = consumer.next()
self.assertEqual(next, list(range(upload_size)))
def __init__(self,
write_key=None,
host=None,
debug=False,
max_queue_size=10000,
send=True,
on_error=None,
flush_at=100,
flush_interval=0.5,
gzip=False,
max_retries=3,
sync_mode=False,
timeout=15,
thread=1):
require('write_key', write_key, string_types)
self.queue = queue.Queue(max_queue_size)
self.write_key = write_key
self.on_error = on_error
self.debug = debug
self.send = send
self.sync_mode = sync_mode
self.host = host
self.gzip = gzip
self.timeout = timeout
if debug:
self.log.setLevel(logging.DEBUG)
if sync_mode:
self.consumers = None
else:
# On program exit, allow the consumer thread to exit cleanly.
# This prevents exceptions and a messy shutdown when the
# interpreter is destroyed before the daemon thread finishes
# execution. However, it is *not* the same as flushing the queue!
# To guarantee all messages have been delivered, you'll still need
# to call flush().
if send:
atexit.register(self.join)
for n in range(thread):
self.consumers = []
consumer = Consumer(
self.queue,
write_key,
host=host,
on_error=on_error,
flush_at=flush_at,
flush_interval=flush_interval,
gzip=gzip,
retries=max_retries,
timeout=timeout,
)
self.consumers.append(consumer)
# if we've disabled sending, just don't start the consumer
if send:
consumer.start()
def test_upload(self):
q = Queue()
consumer = Consumer(q, 'testsecret')
track = {
'type': 'track',
'event': 'python event',
'userId': 'userId'
}
q.put(track)
success = consumer.upload()
self.assertTrue(success)
def __init__(self, write_key=None, debug=False, max_queue_size=10000,
send=True, on_error=None):
require('write_key', write_key, string_types)
self.queue = queue.Queue(max_queue_size)
self.consumer = Consumer(self.queue, write_key, on_error=on_error)
self.write_key = write_key
self.on_error = on_error
self.debug = debug
self.send = send
if debug:
self.log.setLevel(logging.DEBUG)
# if we've disabled sending, just don't start the consumer
if send:
self.consumer.start()
def test_upload_interval(self):
# Put _n_ items in the queue, pausing a little bit more than _upload_interval_
# after each one. The consumer should upload _n_ times.
q = Queue()
upload_interval = 0.3
consumer = Consumer(q, 'testsecret', upload_size=10, upload_interval=upload_interval)
with mock.patch('analytics.consumer.post') as mock_post:
consumer.start()
for i in range(0, 3):
track = {
'type': 'track',
'event': 'python event %d' % i,
'userId': 'userId'
}
q.put(track)
time.sleep(upload_interval * 1.1)
self.assertEqual(mock_post.call_count, 3)
def test_multiple_uploads_per_interval(self):
# Put _upload_size*2_ items in the queue at once, then pause for _upload_interval_.
# The consumer should upload 2 times.
q = Queue()
upload_interval = 0.5
upload_size = 10
consumer = Consumer(q, 'testsecret', upload_size=upload_size, upload_interval=upload_interval)
with mock.patch('analytics.consumer.post') as mock_post:
consumer.start()
for i in range(0, upload_size * 2):
track = {
'type': 'track',
'event': 'python event %d' % i,
'userId': 'userId'
}
q.put(track)
time.sleep(upload_interval * 1.1)
self.assertEqual(mock_post.call_count, 2)
def test_flush_interval(self):
# Put _n_ items in the queue, pausing a little bit more than
# _flush_interval_ after each one.
# The consumer should upload _n_ times.
q = Queue()
flush_interval = 0.3
consumer = Consumer(q, 'testsecret', flush_at=10,
flush_interval=flush_interval)
with mock.patch('analytics.consumer.post') as mock_post:
consumer.start()
for i in range(0, 3):
track = {
'type': 'track',
'event': 'python event %d' % i,
'userId': 'userId'
}
q.put(track)
time.sleep(flush_interval * 1.1)
self.assertEqual(mock_post.call_count, 3)
def test_multiple_uploads_per_interval(self):
# Put _flush_at*2_ items in the queue at once, then pause for
# _flush_interval_. The consumer should upload 2 times.
q = Queue()
flush_interval = 0.5
flush_at = 10
consumer = Consumer(q, 'testsecret', flush_at=flush_at,
flush_interval=flush_interval)
with mock.patch('analytics.consumer.post') as mock_post:
consumer.start()
for i in range(0, flush_at * 2):
track = {
'type': 'track',
'event': 'python event %d' % i,
'userId': 'userId'
}
q.put(track)
time.sleep(flush_interval * 1.1)
self.assertEqual(mock_post.call_count, 2)
def _test_request_retry(self, expected_exception, exception_count):
def mock_post(*args, **kwargs):
mock_post.call_count += 1
if mock_post.call_count <= exception_count:
raise expected_exception
mock_post.call_count = 0
with mock.patch('analytics.consumer.post',
mock.Mock(side_effect=mock_post)):
consumer = Consumer(None, 'testsecret')
track = {
'type': 'track',
'event': 'python event',
'userId': 'userId'
}
# request() should succeed if the number of exceptions raised is less
# than the retries paramater.
if exception_count <= consumer.retries:
consumer.request([track])
else:
# if exceptions are raised more times than the retries parameter,
# we expect the exception to be returned to the caller.
try:
consumer.request([track])
except type(expected_exception) as exc:
self.assertEqual(exc, expected_exception)
else:
self.fail(
"request() should raise an exception if still failing after %d retries"
% consumer.retries)
def test_max_batch_size(self):
q = Queue()
consumer = Consumer(q, 'testsecret', upload_size=100000, upload_interval=3)
track = {
'type': 'track',
'event': 'python event',
'userId': 'userId'
}
msg_size = len(json.dumps(track).encode())
n_msgs = int(475000 / msg_size) # number of messages in a maximum-size batch
def mock_post_fn(_, data, **kwargs):
res = mock.Mock()
res.status_code = 200
self.assertTrue(len(data.encode()) < 500000, 'batch size (%d) exceeds 500KB limit' % len(data.encode()))
return res
with mock.patch('analytics.request._session.post', side_effect=mock_post_fn) as mock_post:
consumer.start()
for _ in range(0, n_msgs + 2):
q.put(track)
q.join()
self.assertEquals(mock_post.call_count, 2)
def __init__(self, write_key=None, debug=False, max_queue_size=10000,
send=True, on_error=None):
require('write_key', write_key, string_types)
self.queue = queue.Queue(max_queue_size)
self.consumer = Consumer(self.queue, write_key, on_error=on_error)
self.write_key = write_key
self.on_error = on_error
self.debug = debug
self.send = send
if debug:
self.log.setLevel('debug')
self.consumer.start()
def __init__(self, write_key=None, debug=False, max_queue_size=10000,
send=True, on_error=None):
require('write_key', write_key, string_types)
self.queue = queue.Queue(max_queue_size)
self.consumer = Consumer(self.queue, write_key, on_error=on_error)
self.write_key = write_key
self.on_error = on_error
self.debug = debug
self.send = send
if debug:
self.log.setLevel('DEBUG')
# if we've disabled sending, just don't start the consumer
if send:
self.consumer.start()
def __init__(self, write_key=None, debug=False, max_queue_size=10000,
send=True, on_error=None):
require('write_key', write_key, string_types)
self.queue = queue.Queue(max_queue_size)
self.consumer = Consumer(self.queue, write_key, on_error=on_error)
self.write_key = write_key
self.on_error = on_error
self.debug = debug
self.send = send
if debug:
self.log.setLevel(logging.DEBUG)
# if we've disabled sending, just don't start the consumer
if send:
# On program exit, allow the consumer thread to exit cleanly.
# This prevents exceptions and a messy shutdown when the interpreter is
# destroyed before the daemon thread finishes execution. However, it
# is *not* the same as flushing the queue! To guarantee all messages
# have been delivered, you'll still need to call flush().
atexit.register(self.join)
self.consumer.start()
class Client(object):
"""Create a new Segment client."""
log = logging.getLogger('segment')
def __init__(self, write_key=None, host=None, debug=False, max_queue_size=10000,
send=True, on_error=None, upload_size=100, upload_interval=0.5,
gzip=False, max_retries=10):
require('write_key', write_key, string_types)
self.queue = queue.Queue(max_queue_size)
self.consumer = Consumer(self.queue, write_key, host=host, on_error=on_error,
upload_size=upload_size, upload_interval=upload_interval,
gzip=gzip, retries=max_retries)
self.write_key = write_key
self.on_error = on_error
self.debug = debug
self.send = send
if debug:
self.log.setLevel(logging.DEBUG)
# if we've disabled sending, just don't start the consumer
if send:
# On program exit, allow the consumer thread to exit cleanly.
# This prevents exceptions and a messy shutdown when the interpreter is
# destroyed before the daemon thread finishes execution. However, it
# is *not* the same as flushing the queue! To guarantee all messages
# have been delivered, you'll still need to call flush().
atexit.register(self.join)
self.consumer.start()
def identify(self, user_id=None, traits=None, context=None, timestamp=None,
anonymous_id=None, integrations=None, message_id=None):
traits = traits or {}
context = context or {}
integrations = integrations or {}
require('user_id or anonymous_id', user_id or anonymous_id, ID_TYPES)
require('traits', traits, dict)
msg = {
'integrations': integrations,
'anonymousId': anonymous_id,
'timestamp': timestamp,
'context': context,
'type': 'identify',
'userId': user_id,
'traits': traits,
'messageId': message_id,
}
return self._enqueue(msg)
def track(self, user_id=None, event=None, properties=None, context=None,
timestamp=None, anonymous_id=None, integrations=None, message_id=None):
properties = properties or {}
context = context or {}
integrations = integrations or {}
require('user_id or anonymous_id', user_id or anonymous_id, ID_TYPES)
require('properties', properties, dict)
require('event', event, string_types)
msg = {
'integrations': integrations,
'anonymousId': anonymous_id,
'properties': properties,
'timestamp': timestamp,
'context': context,
'userId': user_id,
'type': 'track',
'event': event,
'messageId': message_id,
}
return self._enqueue(msg)
def alias(self, previous_id=None, user_id=None, context=None,
timestamp=None, integrations=None, message_id=None):
context = context or {}
integrations = integrations or {}
require('previous_id', previous_id, ID_TYPES)
require('user_id', user_id, ID_TYPES)
msg = {
'integrations': integrations,
'previousId': previous_id,
'timestamp': timestamp,
'context': context,
'userId': user_id,
'type': 'alias',
'messageId': message_id,
}
return self._enqueue(msg)
def group(self, user_id=None, group_id=None, traits=None, context=None,
timestamp=None, anonymous_id=None, integrations=None, message_id=None):
traits = traits or {}
context = context or {}
integrations = integrations or {}
require('user_id or anonymous_id', user_id or anonymous_id, ID_TYPES)
require('group_id', group_id, ID_TYPES)
require('traits', traits, dict)
msg = {
'integrations': integrations,
'anonymousId': anonymous_id,
'timestamp': timestamp,
'groupId': group_id,
'context': context,
'userId': user_id,
'traits': traits,
'type': 'group',
'messageId': message_id,
}
return self._enqueue(msg)
def page(self, user_id=None, category=None, name=None, properties=None,
context=None, timestamp=None, anonymous_id=None,
integrations=None, message_id=None):
properties = properties or {}
context = context or {}
integrations = integrations or {}
require('user_id or anonymous_id', user_id or anonymous_id, ID_TYPES)
require('properties', properties, dict)
if name:
require('name', name, string_types)
if category:
require('category', category, string_types)
msg = {
'integrations': integrations,
'anonymousId': anonymous_id,
'properties': properties,
'timestamp': timestamp,
'category': category,
'context': context,
'userId': user_id,
'type': 'page',
'name': name,
'messageId': message_id,
}
return self._enqueue(msg)
def screen(self, user_id=None, category=None, name=None, properties=None,
context=None, timestamp=None, anonymous_id=None,
integrations=None, message_id=None):
properties = properties or {}
context = context or {}
integrations = integrations or {}
require('user_id or anonymous_id', user_id or anonymous_id, ID_TYPES)
require('properties', properties, dict)
if name:
require('name', name, string_types)
if category:
require('category', category, string_types)
msg = {
'integrations': integrations,
'anonymousId': anonymous_id,
'properties': properties,
'timestamp': timestamp,
'category': category,
'context': context,
'userId': user_id,
'type': 'screen',
'name': name,
'messageId': message_id,
}
return self._enqueue(msg)
def _enqueue(self, msg):
"""Push a new `msg` onto the queue, return `(success, msg)`"""
timestamp = msg['timestamp']
if timestamp is None:
timestamp = datetime.utcnow().replace(tzinfo=tzutc())
message_id = msg.get('messageId')
if message_id is None:
message_id = uuid4()
require('integrations', msg['integrations'], dict)
require('type', msg['type'], string_types)
require('timestamp', timestamp, datetime)
require('context', msg['context'], dict)
# add common
timestamp = guess_timezone(timestamp)
msg['timestamp'] = timestamp.isoformat()
msg['messageId'] = stringify_id(message_id)
msg['context']['library'] = {
'name': 'analytics-python',
'version': VERSION
}
msg['userId'] = stringify_id(msg.get('userId', None))
msg['anonymousId'] = stringify_id(msg.get('anonymousId', None))
msg = clean(msg)
self.log.debug('queueing: %s', msg)
# if send is False, return msg as if it was successfully queued
if not self.send:
return True, msg
try:
self.queue.put(msg, block=False)
self.log.debug('enqueued %s.', msg['type'])
return True, msg
except queue.Full:
self.log.warn('analytics-python queue is full')
return False, msg
def flush(self):
"""Forces a flush from the internal queue to the server"""
queue = self.queue
size = queue.qsize()
queue.join()
# Note that this message may not be precise, because of threading.
self.log.debug('successfully flushed about %s items.', size)
def join(self):
"""Ends the consumer thread once the queue is empty. Blocks execution until finished"""
self.consumer.pause()
try:
self.consumer.join()
except RuntimeError:
# consumer thread has not started
pass
def shutdown(self):
"""Flush all messages and cleanly shutdown the client"""
self.flush()
self.join()
class Client(object):
"""Create a new Segment client."""
log = logging.getLogger('segment')
def __init__(self, write_key=None, debug=False, max_queue_size=10000,
send=True, on_error=None):
require('write_key', write_key, string_types)
self.queue = queue.Queue(max_queue_size)
self.consumer = Consumer(self.queue, write_key, on_error=on_error)
self.write_key = write_key
self.on_error = on_error
self.debug = debug
self.send = send
if debug:
self.log.setLevel('DEBUG')
# if we've disabled sending, just don't start the consumer
if send:
self.consumer.start()
def identify(self, user_id=None, traits={}, context={}, timestamp=None,
anonymous_id=None, integrations={}):
require('user_id or anonymous_id', user_id or anonymous_id, ID_TYPES)
require('traits', traits, dict)
msg = {
'integrations': integrations,
'anonymousId': anonymous_id,
'timestamp': timestamp,
'context': context,
'type': 'identify',
'userId': user_id,
'traits': traits
}
return self._enqueue(msg)
def track(self, user_id=None, event=None, properties={}, context={},
timestamp=None, anonymous_id=None, integrations={}):
require('user_id or anonymous_id', user_id or anonymous_id, ID_TYPES)
require('properties', properties, dict)
require('event', event, string_types)
msg = {
'integrations': integrations,
'anonymousId': anonymous_id,
'properties': properties,
'timestamp': timestamp,
'context': context,
'userId': user_id,
'type': 'track',
'event': event
}
return self._enqueue(msg)
def alias(self, previous_id=None, user_id=None, context={}, timestamp=None,
integrations={}):
require('previous_id', previous_id, ID_TYPES)
require('user_id', user_id, ID_TYPES)
msg = {
'integrations': integrations,
'previousId': previous_id,
'timestamp': timestamp,
'context': context,
'userId': user_id,
'type': 'alias'
}
return self._enqueue(msg)
def group(self, user_id=None, group_id=None, traits={}, context={},
timestamp=None, anonymous_id=None, integrations={}):
require('user_id or anonymous_id', user_id or anonymous_id, ID_TYPES)
require('group_id', group_id, ID_TYPES)
require('traits', traits, dict)
msg = {
'integrations': integrations,
'anonymousId': anonymous_id,
'timestamp': timestamp,
'groupId': group_id,
'context': context,
'userId': user_id,
'traits': traits,
'type': 'group'
}
return self._enqueue(msg)
def page(self, user_id=None, category=None, name=None, properties={},
context={}, timestamp=None, anonymous_id=None, integrations={}):
require('user_id or anonymous_id', user_id or anonymous_id, ID_TYPES)
require('properties', properties, dict)
if name:
require('name', name, string_types)
if category:
require('category', category, string_types)
msg = {
'integrations': integrations,
'anonymousId': anonymous_id,
'properties': properties,
'timestamp': timestamp,
'category': category,
'context': context,
'userId': user_id,
'type': 'page',
'name': name,
}
return self._enqueue(msg)
def screen(self, user_id=None, category=None, name=None, properties={},
context={}, timestamp=None, anonymous_id=None, integrations={}):
require('user_id or anonymous_id', user_id or anonymous_id, ID_TYPES)
require('properties', properties, dict)
if name:
require('name', name, string_types)
if category:
require('category', category, string_types)
msg = {
'integrations': integrations,
'anonymousId': anonymous_id,
'properties': properties,
'timestamp': timestamp,
'category': category,
'context': context,
'userId': user_id,
'type': 'screen',
'name': name,
}
return self._enqueue(msg)
def _enqueue(self, msg):
"""Push a new `msg` onto the queue, return `(success, msg)`"""
timestamp = msg['timestamp']
if timestamp is None:
timestamp = datetime.utcnow().replace(tzinfo=tzutc())
require('integrations', msg['integrations'], dict)
require('type', msg['type'], string_types)
require('timestamp', timestamp, datetime)
require('context', msg['context'], dict)
# add common
timestamp = guess_timezone(timestamp)
msg['timestamp'] = timestamp.isoformat()
msg['messageId'] = str(uuid4())
msg['context']['library'] = {
'name': 'analytics-python',
'version': VERSION
}
clean(msg)
if self.queue.full():
self.log.warn('analytics-python queue is full')
return False, msg
self.queue.put(msg)
self.log.debug('enqueued ' + msg['type'] + '.')
return True, msg
def flush(self):
"""Forces a flush from the internal queue to the server"""
queue = self.queue
size = queue.qsize()
queue.join()
self.log.debug('successfully flushed {0} items.'.format(size))
def test_next(self):
q = Queue()
consumer = Consumer(q, '')
q.put(1)
next = consumer.next()
self.assertEqual(next, [1])
class Client(object):
"""Create a new Segment client."""
log = logging.getLogger('segment')
def __init__(self, write_key=None, debug=False, max_queue_size=10000,
send=True, on_error=None):
require('write_key', write_key, string_types)
self.queue = queue.Queue(max_queue_size)
self.consumer = Consumer(self.queue, write_key, on_error=on_error)
self.write_key = write_key
self.on_error = on_error
self.debug = debug
self.send = send
if debug:
self.log.setLevel(logging.DEBUG)
# if we've disabled sending, just don't start the consumer
if send:
# On program exit, allow the consumer thread to exit cleanly.
# This prevents exceptions and a messy shutdown when the interpreter is
# destroyed before the daemon thread finishes execution. However, it
# is *not* the same as flushing the queue! To guarantee all messages
# have been delivered, you'll still need to call flush().
atexit.register(self.join)
self.consumer.start()
def identify(self, user_id=None, traits=None, context=None, timestamp=None,
anonymous_id=None, integrations=None):
traits = traits or {}
context = context or {}
integrations = integrations or {}
require('user_id or anonymous_id', user_id or anonymous_id, ID_TYPES)
require('traits', traits, dict)
msg = {
'integrations': integrations,
'anonymousId': anonymous_id,
'timestamp': timestamp,
'context': context,
'type': 'identify',
'userId': user_id,
'traits': traits
}
return self._enqueue(msg)
def track(self, user_id=None, event=None, properties=None, context=None,
timestamp=None, anonymous_id=None, integrations=None):
properties = properties or {}
context = context or {}
integrations = integrations or {}
require('user_id or anonymous_id', user_id or anonymous_id, ID_TYPES)
require('properties', properties, dict)
require('event', event, string_types)
msg = {
'integrations': integrations,
'anonymousId': anonymous_id,
'properties': properties,
'timestamp': timestamp,
'context': context,
'userId': user_id,
'type': 'track',
'event': event
}
return self._enqueue(msg)
def alias(self, previous_id=None, user_id=None, context=None,
timestamp=None, integrations=None):
context = context or {}
integrations = integrations or {}
require('previous_id', previous_id, ID_TYPES)
require('user_id', user_id, ID_TYPES)
msg = {
'integrations': integrations,
'previousId': previous_id,
'timestamp': timestamp,
'context': context,
'userId': user_id,
'type': 'alias'
}
return self._enqueue(msg)
def group(self, user_id=None, group_id=None, traits=None, context=None,
timestamp=None, anonymous_id=None, integrations=None):
traits = traits or {}
context = context or {}
integrations = integrations or {}
require('user_id or anonymous_id', user_id or anonymous_id, ID_TYPES)
require('group_id', group_id, ID_TYPES)
require('traits', traits, dict)
msg = {
'integrations': integrations,
'anonymousId': anonymous_id,
'timestamp': timestamp,
'groupId': group_id,
'context': context,
'userId': user_id,
'traits': traits,
'type': 'group'
}
return self._enqueue(msg)
def page(self, user_id=None, category=None, name=None, properties=None,
context=None, timestamp=None, anonymous_id=None,
integrations=None):
properties = properties or {}
context = context or {}
integrations = integrations or {}
require('user_id or anonymous_id', user_id or anonymous_id, ID_TYPES)
require('properties', properties, dict)
if name:
require('name', name, string_types)
if category:
require('category', category, string_types)
msg = {
'integrations': integrations,
'anonymousId': anonymous_id,
'properties': properties,
'timestamp': timestamp,
'category': category,
'context': context,
'userId': user_id,
'type': 'page',
'name': name,
}
return self._enqueue(msg)
def screen(self, user_id=None, category=None, name=None, properties=None,
context=None, timestamp=None, anonymous_id=None,
integrations=None):
properties = properties or {}
context = context or {}
integrations = integrations or {}
require('user_id or anonymous_id', user_id or anonymous_id, ID_TYPES)
require('properties', properties, dict)
if name:
require('name', name, string_types)
if category:
require('category', category, string_types)
msg = {
'integrations': integrations,
'anonymousId': anonymous_id,
'properties': properties,
'timestamp': timestamp,
'category': category,
'context': context,
'userId': user_id,
'type': 'screen',
'name': name,
}
return self._enqueue(msg)
def _enqueue(self, msg):
"""Push a new `msg` onto the queue, return `(success, msg)`"""
timestamp = msg['timestamp']
if timestamp is None:
timestamp = datetime.utcnow().replace(tzinfo=tzutc())
require('integrations', msg['integrations'], dict)
require('type', msg['type'], string_types)
require('timestamp', timestamp, datetime)
require('context', msg['context'], dict)
# add common
timestamp = guess_timezone(timestamp)
msg['timestamp'] = timestamp.isoformat()
msg['messageId'] = str(uuid4())
msg['context']['library'] = {
'name': 'analytics-python',
'version': VERSION
}
msg = clean(msg)
self.log.debug('queueing: %s', msg)
# if send is False, return msg as if it was successfully queued
if not self.send:
return True, msg
try:
self.queue.put(msg, block=False)
self.log.debug('enqueued %s.', msg['type'])
return True, msg
except queue.Full:
self.log.warn('analytics-python queue is full')
return False, msg
def flush(self):
"""Forces a flush from the internal queue to the server"""
queue = self.queue
size = queue.qsize()
queue.join()
# Note that this message may not be precise, because of threading.
self.log.debug('successfully flushed about %s items.', size)
def join(self):
"""Ends the consumer thread once the queue is empty. Blocks execution until finished"""
self.consumer.pause()
try:
self.consumer.join()
except RuntimeError:
# consumer thread has not started
pass
def test_pause(self):
consumer = Consumer(None, 'testsecret')
consumer.pause()
self.assertFalse(consumer.running)
class Client(object):
"""Create a new Segment client."""
log = logging.getLogger('segment')
def __init__(self, write_key=None, debug=False, max_queue_size=10000,
send=True, on_error=None):
require('write_key', write_key, string_types)
self.queue = queue.Queue(max_queue_size)
self.consumer = Consumer(self.queue, write_key, on_error=on_error)
self.write_key = write_key
self.on_error = on_error
self.debug = debug
self.send = send
if debug:
self.log.setLevel(logging.DEBUG)
# if we've disabled sending, just don't start the consumer
if send:
self.consumer.start()
def identify(self, user_id=None, traits={}, context={}, timestamp=None,
anonymous_id=None, integrations={}):
require('user_id or anonymous_id', user_id or anonymous_id, ID_TYPES)
require('traits', traits, dict)
msg = {
'integrations': integrations,
'anonymousId': anonymous_id,
'timestamp': timestamp,
'context': context,
'type': 'identify',
'userId': user_id,
'traits': traits
}
return self._enqueue(msg)
def track(self, user_id=None, event=None, properties={}, context={},
timestamp=None, anonymous_id=None, integrations={}):
require('user_id or anonymous_id', user_id or anonymous_id, ID_TYPES)
require('properties', properties, dict)
require('event', event, string_types)
msg = {
'integrations': integrations,
'anonymousId': anonymous_id,
'properties': properties,
'timestamp': timestamp,
'context': context,
'userId': user_id,
'type': 'track',
'event': event
}
return self._enqueue(msg)
def alias(self, previous_id=None, user_id=None, context={}, timestamp=None,
integrations={}):
require('previous_id', previous_id, ID_TYPES)
require('user_id', user_id, ID_TYPES)
msg = {
'integrations': integrations,
'previousId': previous_id,
'timestamp': timestamp,
'context': context,
'userId': user_id,
'type': 'alias'
}
return self._enqueue(msg)
def group(self, user_id=None, group_id=None, traits={}, context={},
timestamp=None, anonymous_id=None, integrations={}):
require('user_id or anonymous_id', user_id or anonymous_id, ID_TYPES)
require('group_id', group_id, ID_TYPES)
require('traits', traits, dict)
msg = {
'integrations': integrations,
'anonymousId': anonymous_id,
'timestamp': timestamp,
'groupId': group_id,
'context': context,
'userId': user_id,
'traits': traits,
'type': 'group'
}
return self._enqueue(msg)
def page(self, user_id=None, category=None, name=None, properties={},
context={}, timestamp=None, anonymous_id=None, integrations={}):
require('user_id or anonymous_id', user_id or anonymous_id, ID_TYPES)
require('properties', properties, dict)
if name:
require('name', name, string_types)
if category:
require('category', category, string_types)
msg = {
'integrations': integrations,
'anonymousId': anonymous_id,
'properties': properties,
'timestamp': timestamp,
'category': category,
'context': context,
'userId': user_id,
'type': 'page',
'name': name,
}
return self._enqueue(msg)
def screen(self, user_id=None, category=None, name=None, properties={},
context={}, timestamp=None, anonymous_id=None, integrations={}):
require('user_id or anonymous_id', user_id or anonymous_id, ID_TYPES)
require('properties', properties, dict)
if name:
require('name', name, string_types)
if category:
require('category', category, string_types)
msg = {
'integrations': integrations,
'anonymousId': anonymous_id,
'properties': properties,
'timestamp': timestamp,
'category': category,
'context': context,
'userId': user_id,
'type': 'screen',
'name': name,
}
return self._enqueue(msg)
def _enqueue(self, msg):
"""Push a new `msg` onto the queue, return `(success, msg)`"""
timestamp = msg['timestamp']
if timestamp is None:
timestamp = datetime.utcnow().replace(tzinfo=tzutc())
require('integrations', msg['integrations'], dict)
require('type', msg['type'], string_types)
require('timestamp', timestamp, datetime)
require('context', msg['context'], dict)
# add common
timestamp = guess_timezone(timestamp)
msg['timestamp'] = timestamp.isoformat()
msg['messageId'] = str(uuid4())
msg['context']['library'] = {
'name': 'analytics-python',
'version': VERSION
}
msg = clean(msg)
self.log.debug('queueing: %s', msg)
if self.queue.full():
self.log.warn('analytics-python queue is full')
return False, msg
self.queue.put(msg)
self.log.debug('enqueued ' + msg['type'] + '.')
return True, msg
def flush(self):
"""Forces a flush from the internal queue to the server"""
queue = self.queue
size = queue.qsize()
queue.join()
self.log.debug('successfully flushed {0} items.'.format(size))
def join(self):
"""Ends the consumer thread once the queue is empty. Blocks execution until finished"""
self.consumer.pause()
self.consumer.join()
def test_proxies(cls):
consumer = Consumer(None, 'testsecret', proxies='203.243.63.16:80')
track = {'type': 'track', 'event': 'python event', 'userId': 'userId'}
consumer.request([track])
def test_next(self):
q = Queue()
consumer = Consumer(q, '')
q.put(1)
next = consumer.next()
self.assertEqual(next, [1])
def test_pause(self):
consumer = Consumer(None, 'testsecret')
consumer.pause()
self.assertFalse(consumer.running)
