def _get_stats(self, stat_type, stat_range):
if stat_type == 'participations':
stat_type = 'p'
exp_key = self.experiment.name
elif stat_type == 'conversions':
stat_type = 'c'
exp_key = self.experiment.kpi_key()
else:
raise ValueError("Unrecognized stat type: {0}".format(stat_type))
if stat_range not in ['days', 'months', 'years']:
raise ValueError("Unrecognized stat range: {0}".format(stat_range))
stats = {}
pipe = self.redis.pipeline()
search_key = _key("{0}:{1}:{2}".format(stat_type, exp_key, stat_range))
keys = self.redis.smembers(search_key)
for k in keys:
name = self.name if stat_type == 'p' else "{0}:users".format(
self.name)
range_key = _key("{0}:{1}:{2}:{3}".format(stat_type, exp_key, name,
k))
pipe.bitcount(range_key)
redis_results = pipe.execute()
for idx, k in enumerate(keys):
stats[k] = float(redis_results[idx])
return stats
def _get_stats(self, stat_type, stat_range):
if stat_type == 'participations':
stat_type = 'p'
exp_key = self.name
elif stat_type == 'visits':
stat_type = 'v'
exp_key = self.name
elif stat_type == 'conversions':
stat_type = 'c'
exp_key = self.kpi_key()
else:
raise ValueError("Unrecognized stat type: {0}".format(stat_type))
if stat_range not in ['days', 'months', 'years']:
raise ValueError("Unrecognized stat range: {0}".format(stat_range))
pipe = self.redis.pipeline()
stats = {}
search_key = _key("{0}:{1}:{2}".format(stat_type, exp_key, stat_range))
keys = self.redis.smembers(search_key)
for k in keys:
range_key = _key("{0}:{1}:_all:{3}".format(stat_type, self.name, k))
if stat_type == 'p':
pipe.bitcount(range_key)
else:
pipe.get(range_key)
redis_results = pipe.execute()
for idx, k in enumerate(keys):
val = redis_results[idx]
stats[k] = float(val) if val is not None else 0.0
return stats
def _get_stats(self, stat_type, stat_range):
if stat_type == "participations":
stat_type = "p"
exp_key = self.name
elif stat_type == "conversions":
stat_type = "c"
exp_key = self.kpi_key()
else:
raise ValueError("Unrecognized stat type: {0}".format(stat_type))
if stat_range not in ["days", "months", "years"]:
raise ValueError("Unrecognized stat range: {0}".format(stat_range))
pipe = self.redis.pipe()
stats = {}
search_key = _key("{0}:{1}:{2}".format(stat_type, exp_key, stat_range))
keys = self.redis.smembers(search_key)
for k in keys:
mod = "" if stat_type == "p" else "users:"
range_key = _key("{0}:{1}:_all:{2}{3}".format(stat_type, self.name, mod, k))
pipe.bitcount(range_key)
redis_results = pipe.execute()
for idx, k in enumerate(keys):
stats[k] = float(redis_results[idx])
return stats
def _get_stats(self, stat_type, stat_range):
if stat_type == 'participations':
stat_type = 'p'
exp_key = self.experiment.name
elif stat_type == 'conversions':
stat_type = 'c'
exp_key = self.experiment.kpi_key()
else:
raise ValueError("Unrecognized stat type: {0}".format(stat_type))
if stat_range not in ['days', 'months', 'years']:
raise ValueError("Unrecognized stat range: {0}".format(stat_range))
stats = {}
pipe = self.redis.pipeline()
search_key = _key("{0}:{1}:{2}".format(stat_type, exp_key, stat_range))
keys = self.redis.smembers(search_key)
for k in keys:
name = self.name if stat_type == 'p' else "{0}:users".format(self.name)
range_key = _key("{0}:{1}:{2}:{3}".format(stat_type, exp_key, name, k))
pipe.bitcount(range_key)
redis_results = pipe.execute()
for idx, k in enumerate(keys):
stats[k] = float(redis_results[idx])
return stats
def client_alternatives(self, start, end):
seq_id_client_map = self.associated_clients(start, end)
client_id_alternative_map = {}
if not seq_id_client_map:
return {}
sequence_id_list = seq_id_client_map.keys()
experiment_alternatives = self.get_alternative_names()
excluded_key = _key("e:{0}:excluded".format(self.name_key))
alternative_keys = [_key("p:{0}:{1}:all".format(self.name_key, alt)) for alt in experiment_alternatives]
keys = [excluded_key] + alternative_keys
sequences = ','.join(map(str, sequence_id_list))
sequence_alternatives_list = self.redis.eval(
user_experiment_alternatives_script, len(keys),
*(keys + [sequences])
)
if len(sequence_alternatives_list) != len(sequence_id_list):
raise APIError('Unable to fetch alternatives for all clients')
for i in range(len(sequence_alternatives_list)):
sequence_id, alternative_key = sequence_alternatives_list[i]
client_id = seq_id_client_map[sequence_id]
if 'excluded' in alternative_key:
alternative = 'excluded'
else:
alternative = alternative_key.split(':')[-2]
client_id_alternative_map[client_id] = alternative
return client_id_alternative_map
def delete(self):
pipe = self.redis.pipeline()
pipe.srem(_key('e'), self.name)
pipe.delete(self.key())
pipe.delete(_key(self.name))
pipe.delete(_key('e:{0}'.format(self.name)))
# Consider a 'non-keys' implementation of this
keys = self.redis.keys('*:{0}:*'.format(self.name))
for key in keys:
pipe.delete(key)
pipe.execute()
def delete(self):
pipe = self.redis.pipeline()
pipe.srem(_key('e'), self.name)
pipe.delete(self.key())
pipe.delete(_key(self.name))
pipe.delete(_key('e:{0}'.format(self.name)))
# Consider a 'non-keys' implementation of this
keys = self.redis.keys('*:{0}:*'.format(self.name))
for key in keys:
pipe.delete(key)
pipe.execute()
def find(cls, experiment_name, redis_conn):
if not redis_conn.sismember(_key("e"), experiment_name):
raise ValueError('experiment does not exist')
return cls(experiment_name,
Experiment.load_alternatives(experiment_name, redis_conn),
redis_conn)
def save(self):
pipe = self.redis.pipeline()
pipe.watch(self.key())
is_new_record = self.is_new_record()
try:
pipe.multi()
if is_new_record:
pipe.sadd(_key('e'), self.name)
pipe.hset(self.key(), 'created_at',
datetime.now().strftime("%Y-%m-%d %H:%M"))
# reverse here and use lpush to keep consistent with using lrange
for alternative in reversed(self.alternatives):
pipe.lpush("{0}:alternatives".format(self.key()),
alternative.name)
for i in range(len(self.alt_fractions)):
pipe.hset(
self.key(),
"{0}".format(list(reversed(self.alternatives))[i].name),
list(reversed(self.alt_fractions))[i])
pipe.hset(self.key(), 'traffic_fraction', self._traffic_fraction)
# pipe.hset(self.key(),)
pipe.execute()
except redis.WatchError:
# another writer has created this experiment and caused
# our transaction to fail. assume that everything except
# the traffic_fraction is the same between the two writers
# and ensure that the traffic_fraction is updated.
self.redis.hset(self.key(), 'traffic_fraction',
self._traffic_fraction)
def find(cls, experiment_name, redis_conn):
if not redis_conn.sismember(_key("e"), experiment_name):
raise ValueError('experiment does not exist')
return cls(experiment_name,
Experiment.load_alternatives(experiment_name, redis_conn),
redis_conn)
def load_alternatives(experiment_name, redis=None):
"""
:param experiment_name:
:param redis:
:return: 实验对应所有分组
"""
key = _key("e:{0}:alternatives".format(experiment_name))
return redis.lrange(key, 0, -1)
def existing_conversion(self, client):
alts = self.get_alternative_names()
keys = [_key("c:{0}:{1}:users:all".format(self.kpi_key(), alt)) for alt in alts]
altkey = first_key_with_bit_set(keys=keys, args=[self.sequential_id(client)])
if altkey:
idx = keys.index(altkey)
return Alternative(alts[idx], self, redis=self.redis)
return None
def existing_conversion(self, client):
alts = self.get_alternative_names()
keys = [_key("c:{0}:{1}:users:all".format(self.kpi_key(), alt)) for alt in alts]
altkey = first_key_with_bit_set(keys=keys, args=[self.sequential_id(client)])
if altkey:
idx = keys.index(altkey)
return Alternative(alts[idx], self, redis=self.redis)
return None
def find(cls, experiment_name, redis=None):
"""
判断实验是否存在，不存在抛出值异常
"""
if not redis.sismember(_key("e"), experiment_name):
raise ValueError('experiment does not exist')
return cls(experiment_name,
Experiment.load_alternatives(experiment_name, redis),
redis=redis)
def find(cls, experiment_name,
redis=None,
queue = None):
if not redis.sismember(_key("e"), experiment_name):
raise ValueError('experiment does not exist')
return cls(experiment_name,
Experiment.load_alternatives(experiment_name, redis),
redis=redis, queue=queue)
def all(exclude_archived=True, redis=None):
experiments = []
keys = redis.smembers(_key('e'))
for key in keys:
experiment = Experiment.find(key, redis=redis)
if experiment.is_archived() and exclude_archived:
continue
experiments.append(experiment)
return experiments
def all(exclude_archived=True, redis=None):
experiments = []
keys = redis.smembers(_key('e'))
for key in keys:
experiment = Experiment.find(key, redis=redis)
if experiment.is_archived() and exclude_archived:
continue
experiments.append(experiment)
return experiments
def all(redis_conn, exclude_archived=True):
experiments = []
keys = redis_conn.smembers(_key("e"))
for key in keys:
experiment = Experiment.find(key, redis_conn)
if experiment.is_archived() and exclude_archived:
continue
experiments.append(experiment)
return experiments
def associated_clients(self, start=1, end=5000):
'''
returns map of sequence_id to client_id
{
seq_id: "client_id"
}
'''
key = _key('e:{0}:users'.format(self.name_key))
result = self.redis.zrangebyscore(key, start, end, withscores=True)
client_seq_map = dict((int(seq_id), client_id) for client_id, seq_id in result)
return client_seq_map
def all(api_key, exclude_archived=True, exclude_paused=True, redis=None):
experiments = []
keys = redis.smembers(_key('e:{0}'.format(api_key)))
for key in keys:
experiment = Experiment.find(api_key, key, redis=redis)
if experiment.is_archived() and exclude_archived:
continue
if experiment.is_paused() and exclude_paused:
continue
experiments.append(experiment)
return experiments
def save(self):
pipe = self.redis.pipeline()
if self.is_new_record():
pipe.sadd(_key('e'), self.name)
pipe.hset(self.key(), 'created_at', datetime.now().strftime("%Y-%m-%d %H:%M"))
# reverse here and use lpush to keep consistent with using lrange
for alternative in reversed(self.alternatives):
pipe.lpush("{0}:alternatives".format(self.key()), alternative.name)
# allow traffic fraction to change in mid-flight of an experiment.
pipe.hset(self.key(), 'traffic_fraction', self._traffic_fraction)
pipe.execute()
def save(self):
pipe = self.redis.pipeline()
if self.is_new_record():
pipe.sadd(_key('e'), self.name)
pipe.hset(self.key(), 'created_at', datetime.now().strftime("%Y-%m-%d %H:%M"))
# reverse here and use lpush to keep consistent with using lrange
for alternative in reversed(self.alternatives):
pipe.lpush("{0}:alternatives".format(self.key()), alternative.name)
# allow traffic fraction to change in mid-flight of an experiment.
pipe.hset(self.key(), 'traffic_fraction', self._traffic_fraction)
pipe.execute()
def existing_alternative(self, client):
if self.is_client_excluded(client):
return self.control
alts = self.get_alternative_names()
keys = [_key("p:{0}:{1}:all".format(self.name, alt)) for alt in alts]
altkey = first_key_with_bit_set(keys=keys, args=[self.sequential_id(client)])
if altkey:
idx = keys.index(altkey)
return Alternative(alts[idx], self, redis=self.redis)
return None
def existing_alternative(self, client):
if self.is_client_excluded(client):
return self.control
alts = self.get_alternative_names()
keys = [_key("p:{0}:{1}:all".format(self.name, alt)) for alt in alts]
altkey = first_key_with_bit_set(keys=keys, args=[self.sequential_id(client)])
if altkey:
idx = keys.index(altkey)
return Alternative(alts[idx], self, redis=self.redis)
return None
def get_alternative(self, client, dt=None, prefetch=False):
"""Returns and records an alternative according to the following
precedence:
1. An existing alternative
2. A server-chosen alternative
"""
if self.is_archived() or self.is_paused():
return self.control
#pipe.eval(monotonic_zadd_script, 1, _key("e:{0}:users".format(self.name_key)), client.client_id)
self.sequential_id(client)
alts = self.get_alternative_names()
keys = [_key("p:{0}:{1}:all".format(self.name_key, alt)) for alt in alts]
pipe = self.redis.pipeline()
pipe.getbit(_key("e:{0}:excluded".format(self.name_key)), self.sequential_id(client))
pipe.eval(first_key_with_bit_set_script, len(keys), *(keys + [self.sequential_id(client)]))
results = pipe.execute()
is_client_excluded = results[0]
if is_client_excluded:
return self.control
chosen_alternative = None
altkey = results[1]
if altkey:
idx = keys.index(altkey)
chosen_alternative = Alternative(alts[idx], self, redis=self.redis)
if not chosen_alternative:
chosen_alternative, participate = self.choose_alternative(client)
if participate and not prefetch:
gevent.spawn(chosen_alternative.record_participation, client, dt=dt)
gevent.sleep(0)
return chosen_alternative
def save(self):
pipe = self.redis.pipeline()
if self.is_new_record():
pipe.sadd(_key('e'), self.name)
pipe.hset(self.key(), 'created_at', datetime.now())
pipe.hset(self.key(), 'traffic_fraction', self._traffic_fraction)
# reverse here and use lpush to keep consistent with using lrange
for alternative in reversed(self.alternatives):
pipe.lpush("{0}:alternatives".format(self.key()), alternative.name)
pipe.execute()
def save(self):
pipe = self.redis.pipeline()
if self.is_new_record():
pipe.sadd(_key('e'), self.name)
pipe.hset(self.key(), 'created_at', datetime.now())
if self.traffic_dist is not None:
pipe.hset(self.key(), 'traffic_dist', self.traffic_dist)
# reverse here and use lpush to keep consistent with using lrange
for alternative in reversed(self.alternatives):
pipe.lpush("{0}:alternatives".format(self.key()), alternative.name)
pipe.execute()
def save(self):
pipe = self.redis.pipeline()
if self.is_new_record():
pipe.sadd(_key("e"), self.name)
pipe.hset(self.key(), "created_at", datetime.now())
if self.traffic_dist is not None:
pipe.hset(self.key(), "traffic_dist", self.traffic_dist)
# reverse here and use lpush to keep consistent with using lrange
for alternative in reversed(self.alternatives):
pipe.lpush("{0}:alternatives".format(self.key()), alternative.name)
pipe.execute()
def existing_alternative(self, client):
if self.is_client_excluded(client):
return None
alts = self.get_alternative_names()
# print("existing_alternative alts:",alts)
keys = [_key("p:{0}:{1}:all".format(self.name, alt)) for alt in alts]
# print("existing_alternative keys:",keys)
# print("existing_alternative self.sequential_id(client):",self.sequential_id(client))
altkey = first_key_with_bit_set(keys=keys,
args=[self.sequential_id(client)])
# print("excluded_clients altkey:",altkey)
if altkey:
idx = keys.index(altkey)
# print("existing_alternative keys.index:",idx)
return Alternative(alts[idx], self, redis=self.redis)
return None
def save(self):
pipe = self.redis.pipeline()
pipe.watch(self.key())
is_new_record = self.is_new_record()
try:
pipe.multi()
if is_new_record:
pipe.sadd(_key('e'), self.name)
pipe.hset(self.key(), 'created_at', datetime.now().strftime("%Y-%m-%d %H:%M"))
# reverse here and use lpush to keep consistent with using lrange
for alternative in reversed(self.alternatives):
pipe.lpush("{0}:alternatives".format(self.key()), alternative.name)
pipe.hset(self.key(), 'traffic_fraction', self._traffic_fraction)
pipe.execute()
except redis.WatchError:
# another writer has created this experiment and caused
# our transaction to fail. assume that everything except
# the traffic_fraction is the same between the two writers
# and ensure that the traffic_fraction is updated.
self.redis.hset(self.key(), 'traffic_fraction', self._traffic_fraction)
def find(cls, api_key, experiment_name,
redis=None):
ekey = "e:{0}:{1}".format(api_key, experiment_name)
pipe = redis.pipeline()
pipe.sismember(_key("a"), api_key)
pipe.sismember(_key("e:{0}".format(api_key)), experiment_name)
pipe.lrange(_key("e:{0}:{1}:alternatives".format(api_key, experiment_name)), 0, -1)
pipe.hget(_key(ekey), 'traffic_fraction')
pipe.get(_key(ekey+':winner'))
pipe.hexists(_key(ekey), 'archived')
pipe.hexists(_key(ekey), 'paused')
results = pipe.execute()
if not results[0]:
raise APIError('API Key does not exists')
if not results[1]:
raise ValueError('experiment does not exist')
return cls(api_key, experiment_name,
results[2], traffic_fraction=float(results[3]), winner=results[4],
redis=redis, is_archived=results[5], is_paused=results[6])
def total_participants(self):
key = _key("p:{0}:_all:all".format(self.name))
return self.redis.bitcount(key)
def total_conversions(self):
key = _key("c:{0}:_all:users:all".format(self.kpi_key()))
return self.redis.bitcount(key)
def archive(self):
self.redis.hset(self.key(), 'archived', 1)
self.redis.delete(_key('e:{0}:users'.format(self.name)))
def total_conversions(self):
key = _key("c:{0}:_all:users:all".format(self.kpi_key()))
return self.redis.bitcount(key)
def all_names(redis=None):
return redis.smembers(_key('e'))
def participant_count(self):
key = _key("p:{0}:{1}:all".format(self.experiment.name, self.name))
return self.redis.bitcount(key)
def load_alternatives(experiment_name, redis=None):
key = _key("e:{0}:alternatives".format(experiment_name))
return redis.lrange(key, 0, -1)
def excluded_clients(self):
key = _key("e:{0}:excluded".format(self.name))
return self.redis.bitcount(key)
def key(self):
return _key("{0}:{1}".format(self.experiment.name, self.name))
def total_participants(self):
key = _key("p:{0}:_all:all".format(self.name))
return self.redis.bitcount(key)
def record_conversion(self, client, dt=None):
"""Record a user's conversion in a test along with a given variation"""
if dt is None:
date = datetime.now()
else:
date = dt
experiment_key = self.experiment.kpi_key()
pipe = self.redis.pipeline()
pipe.sadd(_key("c:{0}:years".format(experiment_key)), date.strftime('%Y'))
pipe.sadd(_key("c:{0}:months".format(experiment_key)), date.strftime('%Y-%m'))
pipe.sadd(_key("c:{0}:days".format(experiment_key)), date.strftime('%Y-%m-%d'))
pipe.execute()
keys = [
_key("c:{0}:_all:users:all".format(experiment_key)),
_key("c:{0}:_all:users:{1}".format(experiment_key, date.strftime('%Y'))),
_key("c:{0}:_all:users:{1}".format(experiment_key, date.strftime('%Y-%m'))),
_key("c:{0}:_all:users:{1}".format(experiment_key, date.strftime('%Y-%m-%d'))),
_key("c:{0}:{1}:users:all".format(experiment_key, self.name)),
_key("c:{0}:{1}:users:{2}".format(experiment_key, self.name, date.strftime('%Y'))),
_key("c:{0}:{1}:users:{2}".format(experiment_key, self.name, date.strftime('%Y-%m'))),
_key("c:{0}:{1}:users:{2}".format(experiment_key, self.name, date.strftime('%Y-%m-%d'))),
]
msetbit(keys=keys, args=([self.experiment.sequential_id(client), 1] * len(keys)))
def is_new_record(self):
return not self.redis.sismember(_key("e"), self.name)
def completed_count(self):
key = _key("c:{0}:{1}:users:all".format(self.experiment.kpi_key(), self.name))
return self.redis.bitcount(key)
def exclude_client(self, client):
key = _key("e:{0}:excluded".format(self.name))
self.redis.setbit(key, self.sequential_id(client), 1)
def completed_count(self):
key = _key("c:{0}:{1}:users:all".format(self.experiment.kpi_key(),
self.name))
return self.redis.bitcount(key)
def is_client_excluded(self, client):
key = _key("e:{0}:excluded".format(self.name))
return self.redis.getbit(key, self.sequential_id(client))
def exclude_client(self, client):
key = _key("e:{0}:excluded".format(self.name))
self.redis.setbit(key, self.sequential_id(client), 1)
def key(self, include_kpi=True):
if include_kpi:
return _key("e:{0}".format(self.kpi_key()))
else:
return _key("e:{0}".format(self.name))
def record_conversion(self, client, dt=None):
"""Record a user's conversion in a test along with a given variation"""
if dt is None:
date = datetime.now()
else:
date = dt
experiment_key = self.experiment.kpi_key()
pipe = self.redis.pipeline()
pipe.sadd(_key("c:{0}:years".format(experiment_key)),
date.strftime('%Y'))
pipe.sadd(_key("c:{0}:months".format(experiment_key)),
date.strftime('%Y-%m'))
pipe.sadd(_key("c:{0}:days".format(experiment_key)),
date.strftime('%Y-%m-%d'))
pipe.execute()
keys = [
_key("c:{0}:_all:users:all".format(experiment_key)),
_key("c:{0}:_all:users:{1}".format(experiment_key,
date.strftime('%Y'))),
_key("c:{0}:_all:users:{1}".format(experiment_key,
date.strftime('%Y-%m'))),
_key("c:{0}:_all:users:{1}".format(experiment_key,
date.strftime('%Y-%m-%d'))),
_key("c:{0}:{1}:users:all".format(experiment_key, self.name)),
_key("c:{0}:{1}:users:{2}".format(experiment_key, self.name,
date.strftime('%Y'))),
_key("c:{0}:{1}:users:{2}".format(experiment_key, self.name,
date.strftime('%Y-%m'))),
_key("c:{0}:{1}:users:{2}".format(experiment_key, self.name,
date.strftime('%Y-%m-%d'))),
]
msetbit(keys=keys,
args=([self.experiment.sequential_id(client), 1] * len(keys)))
def all_names(redis_conn):
return redis_conn.smembers(_key('e'))
def load_alternatives(experiment_name, redis=None):
key = _key("e:{0}:alternatives".format(experiment_name))
return redis.lrange(key, 0, -1)
def participant_count(self):
key = _key("p:{0}:{1}:all".format(self.experiment.name, self.name))
return self.redis.bitcount(key)
def excluded_clients(self):
key = _key("e:{0}:excluded".format(self.name))
return self.redis.bitcount(key)
def key(self, include_kpi=True):
if include_kpi:
return _key("e:{0}".format(self.kpi_key()))
else:
return _key("e:{0}".format(self.name))
def is_client_excluded(self, client):
key = _key("e:{0}:excluded".format(self.name))
return self.redis.getbit(key, self.sequential_id(client))
def key(self):
return _key("{0}:{1}".format(self.experiment.name, self.name))
def all_names(redis=None):
return redis.smembers(_key('e'))
