def test_get_cacheable_result(self):
self.create_contact("Bob", number="1234")
def calculate():
return Contact.objects.all().count()
with self.assertNumQueries(1):
self.assertEqual(
get_cacheable_result('test_contact_count', 60, calculate),
1) # from db
with self.assertNumQueries(0):
self.assertEqual(
get_cacheable_result('test_contact_count', 60, calculate),
1) # from cache
self.create_contact("Jim", number="2345")
with self.assertNumQueries(0):
self.assertEqual(
get_cacheable_result('test_contact_count', 60, calculate),
1) # not updated
get_redis_connection().delete(
'test_contact_count'
) # delete from cache for force re-fetch from db
with self.assertNumQueries(1):
self.assertEqual(
get_cacheable_result('test_contact_count', 60, calculate),
2) # from db
with self.assertNumQueries(0):
self.assertEqual(
get_cacheable_result('test_contact_count', 60, calculate),
2) # from cache
def clear_all():
if not settings.DEBUG:
raise Exception("Can't clear everything when not in DEBUG mode, dude")
get_redis_connection().flushdb()
base_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
db_path = os.path.join(base_dir, "cg/sqlite3.db")
os.remove(db_path)
def distr_object(t_object, **kwargs):
redis_c = get_redis_connection('feed_storage')
rfeed_list = kwargs.get('feed_list')
rt_object = pickle.dumps(t_object)
if t_object.get_verbtext() == 'updatefcard':
redis_fa = get_redis_connection('favlist')
for i in rfeed_list:
action = redis_fa.hget(str(i), t_object.get_mobject_id())
if action.startswith('A') or (action.startswith('B') and str(i) in action):
redis_c.lpush(str(i), rt_object)
else:
for i in rfeed_list:
redis_c.lpush(i, rt_object)
def distr_object(t_object, **kwargs):
redis_c = get_redis_connection('feed_storage')
rfeed_list = kwargs.get('feed_list')
rt_object = pickle.dumps(t_object)
if t_object.get_verbtext() == 'updatefcard':
redis_fa = get_redis_connection('favlist')
for i in rfeed_list:
action = redis_fa.hget(str(i), t_object.get_mobject_id())
if action.startswith('A') or (action.startswith('B')
and str(i) in action):
redis_c.lpush(str(i), rt_object)
else:
for i in rfeed_list:
redis_c.lpush(i, rt_object)
def get_models(prefix, tag):
"""
Возвращает id-шники элементов с тегом tag
Например все вопросы, у которых есть какой-то тег
"""
r = get_redis_connection()
return r.smembers(tag_key(prefix, tag))
def get_top(max_tags=10, offset=0):
r = get_redis_connection()
# zrevrage возвращает из упорядоченного множества
# первые max_tags тегов начиная с offset
# значения отсортированы (в данном случае по кол-ву вопросов
# в которых они использованы)
return r.zrevrange(key_all_tags, offset, max_tags)
def get_unread_msg(request):
msg_redis = get_redis_connection('message')
if request.user.is_anonymous() == False and request.user.is_authenticated(
):
umsg = msg_redis.get('unread_' + str(request.user.id))
return True if umsg is not None and int(umsg) > 0 else False
def invalidate_cache(cls, contact_field=None, ruleset=None, group=None):
"""
Used to invalidate our summary cache for values. Callers should pass in one (and only one) of a contact field,
ruleset or group that changed and all result summaries that have changed will be invalidated accordingly.
:return: how many cached records were invalidated
"""
if not contact_field and not ruleset and not group:
raise Exception(
"You must specify a contact field, ruleset or group to invalidate results for"
)
if contact_field:
key = ':' + (CONTACT_KEY % contact_field.id) + ':'
elif group:
key = ':' + (GROUP_KEY % group.id) + ':'
elif ruleset:
key = ':' + (RULESET_KEY % ruleset.id) + ':'
# blow away any redis items that contain our key as a dependency
r = get_redis_connection()
keys = r.keys(VALUE_SUMMARY_CACHE_KEY + "*" + key + "*")
if keys:
invalidated = r.delete(*keys)
else:
invalidated = 0
return invalidated
def squash_channelcounts():
r = get_redis_connection()
key = 'squash_channelcounts'
if not r.get(key):
with r.lock(key, timeout=900):
ChannelCount.squash_counts()
def invalidate_cache(cls, contact_field=None, ruleset=None, group=None):
"""
Used to invalidate our summary cache for values. Callers should pass in one (and only one) of a contact field,
ruleset or group that changed and all result summaries that have changed will be invalidated accordingly.
:return: how many cached records were invalidated
"""
if not contact_field and not ruleset and not group:
raise Exception("You must specify a contact field, ruleset or group to invalidate results for")
if contact_field:
key = ':' + (CONTACT_KEY % contact_field.id) + ':'
elif group:
key = ':' + (GROUP_KEY % group.id) + ':'
elif ruleset:
key = ':' + (RULESET_KEY % ruleset.id) + ':'
# blow away any redis items that contain our key as a dependency
r = get_redis_connection()
keys = r.keys(VALUE_SUMMARY_CACHE_KEY + "*" + key + "*")
if keys:
invalidated = r.delete(*keys)
else:
invalidated = 0
return invalidated
def squash_systemlabels():
r = get_redis_connection()
key = 'squash_systemlabels'
if not r.get(key):
with r.lock(key, timeout=900):
SystemLabel.squash_counts()
def check_campaigns_task(sched_id=None):
"""
See if any event fires need to be triggered
"""
logger = check_campaigns_task.get_logger()
# get a lock
r = get_redis_connection()
key = 'check_campaigns'
# only do this if we aren't already checking campaigns
if not r.get(key):
with r.lock(key, timeout=3600):
# for each that needs to be fired
for fire in EventFire.objects.filter(
fired=None, scheduled__lte=timezone.now()):
try:
key = 'fire_campaign_%d' % fire.pk
if not r.get(key):
# try to acquire a lock
with r.lock('fire_campaign_%d' % fire.pk, timeout=120):
# reload it
fire = EventFire.objects.get(id=fire.pk)
if not fire.fired:
fire.fire()
except: # pragma: no cover
logger.error("Error running campaign event: %s" % fire.pk,
exc_info=True)
def process_message_task(msg_id, from_mage=False, new_contact=False):
"""
Processes a single incoming message through our queue.
"""
r = get_redis_connection()
msg = Msg.current_messages.filter(pk=msg_id, status=PENDING).select_related('org', 'contact', 'contact_urn', 'channel').first()
# somebody already handled this message, move on
if not msg:
return
# get a lock on this contact, we process messages one by one to prevent odd behavior in flow processing
key = 'pcm_%d' % msg.contact_id
if not r.get(key):
with r.lock(key, timeout=120):
print "M[%09d] Processing - %s" % (msg.id, msg.text)
start = time.time()
# if message was created in Mage...
if from_mage:
mage_handle_new_message(msg.org, msg)
if new_contact:
mage_handle_new_contact(msg.org, msg.contact)
Msg.process_message(msg)
print "M[%09d] %08.3f s - %s" % (msg.id, time.time() - start, msg.text)
def check_campaigns_task(sched_id=None):
"""
See if any event fires need to be triggered
"""
logger = check_campaigns_task.get_logger()
# get a lock
r = get_redis_connection()
key = 'check_campaigns'
# only do this if we aren't already checking campaigns
if not r.get(key):
with r.lock(key, timeout=3600):
# for each that needs to be fired
for fire in EventFire.objects.filter(
fired=None, scheduled__lte=timezone.now()).select_related(
'contact', 'contact__org'):
try:
push_task(fire.contact.org, HANDLER_QUEUE,
HANDLE_EVENT_TASK,
dict(type=FIRE_EVENT, id=fire.id))
except Exception: # pragma: no cover
logger.error("Error running campaign event: %s" % fire.pk,
exc_info=True)
def handle_event_task():
"""
Priority queue task that handles both event fires (when fired) and new incoming
messages that need to be handled.
Currently two types of events may be "popped" from our queue:
msg - Which contains the id of the Msg to be processed
fire - Which contains the id of the EventFire that needs to be fired
"""
from temba.campaigns.models import EventFire
r = get_redis_connection()
# pop off the next task
event_task = pop_task(HANDLE_EVENT_TASK)
# it is possible we have no message to send, if so, just return
if not event_task:
return
if event_task['type'] == MSG_EVENT:
process_message_task(event_task['id'],
event_task.get('from_mage', False),
event_task.get('new_contact', False))
elif event_task['type'] == FIRE_EVENT:
# use a lock to make sure we don't do two at once somehow
with r.lock('fire_campaign_%s' % event_task['id'], timeout=120):
event = EventFire.objects.filter(pk=event_task['id'],
fired=None).first()
if event:
event.fire()
else:
raise Exception("Unexpected event type: %s" % event_task)
def get_or_open(cls, org, user, labels, message, summary, assignee, archive_messages=True):
r = get_redis_connection()
with r.lock('org:%d:cases_lock' % org.pk):
# check for open case with this contact
existing_open = cls.get_open_for_contact_on(org, message.contact, timezone.now())
if existing_open:
existing_open.is_new = False
return existing_open
# check for another case (possibly closed) connected to this message
existing_for_msg = cls.objects.filter(message_id=message.id).first()
if existing_for_msg:
existing_for_msg.is_new = False
return existing_for_msg
case = cls.objects.create(org=org, assignee=assignee, contact_uuid=message.contact,
summary=summary, message_id=message.id, message_on=message.created_on)
case.is_new = True
case.labels.add(*labels)
CaseAction.create(case, user, CaseAction.OPEN, assignee=assignee)
# archive messages any labelled messages from this contact
if archive_messages:
Contact.archive_messages(org, message.contact)
return case
def squash_flowruncounts():
r = get_redis_connection()
key = 'squash_flowruncounts'
if not r.get(key):
with r.lock(key, timeout=900):
FlowRunCount.squash_counts()
def squash_topupcredits():
r = get_redis_connection()
key = 'squash_topupcredits'
if not r.get(key):
with r.lock(key, timeout=900):
TopUpCredits.squash_credits()
def process_run_timeout(run_id, timeout_on):
"""
Processes a single run timeout
"""
from temba.flows.models import FlowRun
r = get_redis_connection()
run = FlowRun.objects.filter(id=run_id, is_active=True, flow__is_active=True).first()
if run:
key = 'pcm_%d' % run.contact_id
if not r.get(key):
with r.lock(key, timeout=120):
print "T[%09d] Processing timeout" % run.id
start = time.time()
run.refresh_from_db()
# this is still the timeout to process (json doesn't have microseconds so close enough)
if run.timeout_on and abs(run.timeout_on - timeout_on) < timedelta(milliseconds=1):
run.resume_after_timeout()
else:
print "T[%09d] .. skipping timeout, already handled" % run.id
print "T[%09d] %08.3f s" % (run.id, time.time() - start)
def push_task(org, queue, task_name, args, priority=DEFAULT_PRIORITY):
"""
Adds a task to queue_name with the supplied arguments.
Ex: add_task(nyaruka, 'flows', 'start_flow', [1,2,3,4,5,6,7,8,9,10])
"""
r = get_redis_connection('default')
# calculate our score from the current time and priority, this could get us in trouble
# if things are queued for more than ~100 days, but otherwise gives us the properties of prioritizing
# first based on priority, then insertion order.
score = time.time() + priority
# push our task onto the right queue and make sure it is in the active list (atomically)
with r.pipeline() as pipe:
key = "%s:%d" % (task_name, org.id)
pipe.zadd(key, dict_to_json(args), score)
# and make sure this key is in our list of queues so this job will get worked on
pipe.sadd("%s:active" % task_name, key)
pipe.execute()
# if we were given a queue to schedule on, then add this task to celery.
#
# note that the task that is fired needs no arguments as it should just use pop_task with the
# task name to determine what to work on.
if queue:
if getattr(settings, 'CELERY_ALWAYS_EAGER', False):
task_function = lookup_task_function(task_name)
task_function()
else:
current_app.send_task(task_name, args=[], kwargs={}, queue=queue)
def pop_task(task_name):
"""
Pops the next 'random' task off our queue, returning the arguments that were saved
Ex: pop_next_task('start_flow')
<<< {flow=5, contacts=[1,2,3,4,5,6,7,8,9,10]}
"""
r = get_redis_connection('default')
task = None
active_set = "%s:active" % task_name
# get what queue we will work against
queue = r.srandmember(active_set)
while queue:
# this lua script does both a "zpop" (popping the next highest thing off our sorted set) and
# a clearing of our active set if there is no value in it as an atomic action
lua = "local val = redis.call('zrange', ARGV[2], 0, 0) \n" \
"if next(val) == nil then redis.call('srem', ARGV[1], ARGV[2]) return nil \n"\
"else redis.call('zremrangebyrank', ARGV[2], 0, 0) return val[1] end\n"
task = r.eval(lua, 2, 'active_set', 'queue', active_set, queue)
# found a task? then break out
if task is not None:
task = json.loads(task)
break
# if we didn't get a task, then run again against a new queue until there is nothing left in our task queue
queue = r.srandmember(active_set)
return task
def invalidate_cache(cls, contact_field=None, ruleset=None, group=None):
"""
Used to invalidate our summary cache for values. Callers should pass in one (and only one) of a contact field,
ruleset or group that changed and all result summaries that have changed will be invalidated accordingly.
:return: how many cached records were invalidated
"""
if not contact_field and not ruleset and not group:
raise ValueError("You must specify a contact field, ruleset or group to invalidate results for")
if contact_field:
key = CONTACT_KEY % contact_field.id
elif group:
key = GROUP_KEY % group.id
elif ruleset:
key = RULESET_KEY % ruleset.id
# blow away any redis items that contain our key as a dependency
r = get_redis_connection()
dependent_results = r.smembers(key)
# save ourselves a roundtrip if there are no matches
if dependent_results:
# clear all our dependencies
pipe = r.pipeline()
pipe.srem(key, *dependent_results)
pipe.delete(*dependent_results)
pipe.execute()
return len(dependent_results)
def squash_topupcredits():
r = get_redis_connection()
key = 'squash_topupcredits'
if not r.get(key):
with r.lock(key, timeout=900):
TopUpCredits.squash_credits()
def squash_systemlabels():
r = get_redis_connection()
key = 'squash_systemlabels'
if not r.get(key):
with r.lock(key, timeout=900):
SystemLabel.squash_counts()
def invalidate_cache(cls, contact_field=None, ruleset=None, group=None):
"""
Used to invalidate our summary cache for values. Callers should pass in one (and only one) of a contact field,
ruleset or group that changed and all result summaries that have changed will be invalidated accordingly.
:return: how many cached records were invalidated
"""
if not contact_field and not ruleset and not group:
raise Exception(
"You must specify a contact field, ruleset or group to invalidate results for"
)
if contact_field:
key = CONTACT_KEY % contact_field.id
elif group:
key = GROUP_KEY % group.id
elif ruleset:
key = RULESET_KEY % ruleset.id
# blow away any redis items that contain our key as a dependency
r = get_redis_connection()
dependent_results = r.smembers(key)
# save ourselves a roundtrip if there are no matches
if dependent_results:
# clear all our dependencies
pipe = r.pipeline()
pipe.srem(key, *dependent_results)
pipe.delete(*dependent_results)
pipe.execute()
return len(dependent_results)
def handle_event_task():
"""
Priority queue task that handles both event fires (when fired) and new incoming
messages that need to be handled.
Currently two types of events may be "popped" from our queue:
msg - Which contains the id of the Msg to be processed
fire - Which contains the id of the EventFire that needs to be fired
"""
from temba.campaigns.models import EventFire
r = get_redis_connection()
# pop off the next task
event_task = pop_task(HANDLE_EVENT_TASK)
# it is possible we have no message to send, if so, just return
if not event_task:
return
if event_task['type'] == MSG_EVENT:
process_message_task(event_task['id'], event_task.get('from_mage', False), event_task.get('new_contact', False))
elif event_task['type'] == FIRE_EVENT:
# use a lock to make sure we don't do two at once somehow
with r.lock('fire_campaign_%s' % event_task['id'], timeout=120):
event = EventFire.objects.filter(pk=event_task['id'], fired=None).first()
if event:
event.fire()
else:
raise Exception("Unexpected event type: %s" % event_task)
def get_or_open(cls, org, user, message, summary, assignee):
from casepro.profiles.models import Notification
r = get_redis_connection()
with r.lock(CASE_LOCK_KEY % (org.pk, message.contact.uuid)):
message.refresh_from_db()
# if message is already associated with a case, return that
if message.case:
message.case.is_new = False
return message.case
# suspend from groups, expire flows and archive messages
message.contact.prepare_for_case()
case = cls.objects.create(org=org, assignee=assignee, initial_message=message, contact=message.contact,
summary=summary)
case.is_new = True
case.labels.add(*list(message.labels.all())) # copy labels from message to new case
case.watchers.add(user)
# attach message to this case
message.case = case
message.save(update_fields=('case',))
action = CaseAction.create(case, user, CaseAction.OPEN, assignee=assignee)
for assignee_user in assignee.get_users():
if assignee_user != user:
Notification.new_case_assignment(org, assignee_user, action)
return case
def push_task(org, queue, task, args, priority=DEFAULT_PRIORITY):
"""
Adds a task to queue_name with the supplied arguments.
Ex: add_task(nyaruka, 'flows', 'start_flow', [1,2,3,4,5,6,7,8,9,10])
"""
r = get_redis_connection('default')
# calculate our score from the current time and priority, this could get us in trouble
# if things are queued for more than ~100 days, but otherwise gives us the properties of prioritizing
# first based on priority, then insertion order.
score = time.time() + priority
# push our task onto the right queue and make sure it is in the active list (atomically)
with r.pipeline() as pipe:
key = "%s:%d" % (task, org.id)
pipe.zadd(key, dict_to_json(args), score)
# and make sure this key is in our list of queues so this job will get worked on
pipe.sadd("%s:active" % task, key)
pipe.execute()
# if we were given a queue to schedule on, then add this task to celery.
#
# note that the task that is fired needs no arguments as it should just use pop_task with the
# task name to determine what to work on.
if queue:
if getattr(settings, 'CELERY_ALWAYS_EAGER', False):
task_function = lookup_task_function(task)
task_function()
else:
current_app.send_task(task, args=[], kwargs={}, queue=queue)
def pop_task(task_name):
"""
Pops the next 'random' task off our queue, returning the arguments that were saved
Ex: pop_next_task('start_flow')
<<< {flow=5, contacts=[1,2,3,4,5,6,7,8,9,10]}
"""
r = get_redis_connection('default')
task = None
active_set = "%s:active" % task_name
# get what queue we will work against
queue = r.srandmember(active_set)
while queue:
# this lua script does both a "zpop" (popping the next highest thing off our sorted set) and
# a clearing of our active set if there is no value in it as an atomic action
lua = "local val = redis.call('zrange', ARGV[2], 0, 0) \n" \
"if next(val) == nil then redis.call('srem', ARGV[1], ARGV[2]) return nil \n"\
"else redis.call('zremrangebyrank', ARGV[2], 0, 0) return val[1] end\n"
task = r.eval(lua, 2, 'active_set', 'queue', active_set, queue)
# found a task? then break out
if not task is None:
task = json.loads(task)
break
# if we didn't get a task, then run again against a new queue until there is nothing left in our task queue
queue = r.srandmember(active_set)
return task
def send_msg_task():
"""
Pops the next message off of our msg queue to send.
"""
# pop off the next task
msg_tasks = pop_task(SEND_MSG_TASK)
# it is possible we have no message to send, if so, just return
if not msg_tasks:
return
if not isinstance(msg_tasks, list):
msg_tasks = [msg_tasks]
r = get_redis_connection()
# acquire a lock on our contact to make sure two sets of msgs aren't being sent at the same time
try:
with r.lock('send_contact_%d' % msg_tasks[0]['contact'], timeout=300):
# send each of our msgs
while msg_tasks:
msg_task = msg_tasks.pop(0)
msg = dict_to_struct('MockMsg', msg_task,
datetime_fields=['modified_on', 'sent_on', 'created_on', 'queued_on', 'next_attempt'])
Channel.send_message(msg)
# if there are more messages to send for this contact, sleep a second before moving on
if msg_tasks:
time.sleep(1)
finally: # pragma: no cover
# if some msgs weren't sent for some reason, then requeue them for later sending
if msg_tasks:
# requeue any unsent msgs
push_task(msg_tasks[0]['org'], MSG_QUEUE, SEND_MSG_TASK, msg_tasks)
def squash_channelcounts():
r = get_redis_connection()
key = 'squash_channelcounts'
if not r.get(key):
with r.lock(key, timeout=900):
ChannelCount.squash_counts()
def check_campaigns_task(sched_id=None):
"""
See if any event fires need to be triggered
"""
logger = check_campaigns_task.get_logger()
# get a lock
r = get_redis_connection()
key = "check_campaigns"
# only do this if we aren't already checking campaigns
if not r.get(key):
with r.lock(key, timeout=3600):
# for each that needs to be fired
for fire in EventFire.objects.filter(fired=None, scheduled__lte=timezone.now()):
try:
key = "fire_campaign_%d" % fire.pk
if not r.get(key):
# try to acquire a lock
with r.lock("fire_campaign_%d" % fire.pk, timeout=120):
# reload it
fire = EventFire.objects.get(id=fire.pk)
if not fire.fired:
fire.fire()
except: # pragma: no cover
logger.error("Error running campaign event: %s" % fire.pk, exc_info=True)
def process_message_task(msg_id, from_mage=False, new_contact=False):
"""
Processes a single incoming message through our queue.
"""
r = get_redis_connection()
msg = Msg.objects.filter(pk=msg_id, status=PENDING).select_related(
'org', 'contact', 'contact_urn', 'channel').first()
# somebody already handled this message, move on
if not msg:
return
# get a lock on this contact, we process messages one by one to prevent odd behavior in flow processing
key = 'pcm_%d' % msg.contact_id
if not r.get(key):
with r.lock(key, timeout=120):
print "M[%09d] Processing - %s" % (msg.id, msg.text)
start = time.time()
# if message was created in Mage...
if from_mage:
mage_handle_new_message(msg.org, msg)
if new_contact:
mage_handle_new_contact(msg.org, msg.contact)
Msg.process_message(msg)
print "M[%09d] %08.3f s - %s" % (msg.id, time.time() - start,
msg.text)
def purge_broadcasts_task():
"""
Looks for broadcasts older than 90 days and marks their messages as purged
"""
r = get_redis_connection()
# 90 days ago
purge_date = timezone.now() - timedelta(days=90)
key = 'purge_broadcasts_task'
if not r.get(key):
with r.lock(key, timeout=900):
# determine which broadcasts are old
broadcasts = Broadcast.objects.filter(created_on__lt=purge_date,
purged=False)
for broadcast in broadcasts:
# TODO actually delete messages!
#
# Need to also create Debit objects for topups associated with messages being deleted, and then
# regularly squash those.
broadcast.purged = True
broadcast.save(update_fields=['purged'])
def process_run_timeout(run_id, timeout_on):
"""
Processes a single run timeout
"""
from temba.flows.models import FlowRun
r = get_redis_connection()
run = FlowRun.objects.filter(id=run_id,
is_active=True,
flow__is_active=True).first()
if run:
key = 'pcm_%d' % run.contact_id
if not r.get(key):
with r.lock(key, timeout=120):
print "T[%09d] Processing timeout" % run.id
start = time.time()
run.refresh_from_db()
# this is still the timeout to process (json doesn't have microseconds so close enough)
if run.timeout_on and abs(run.timeout_on -
timeout_on) < timedelta(
milliseconds=1):
run.resume_after_timeout()
else:
print "T[%09d] .. skipping timeout, already handled" % run.id
print "T[%09d] %08.3f s" % (run.id, time.time() - start)
def squash_flowruncounts():
r = get_redis_connection()
key = 'squash_flowruncounts'
if not r.get(key):
with r.lock(key, timeout=900):
FlowRunCount.squash_counts()
def get_or_open(cls, org, user, message, summary, assignee):
r = get_redis_connection()
with r.lock(CASE_LOCK_KEY % (org.pk, message.contact.uuid)):
# if message is already associated with a case, return that
if message.case:
message.case.is_new = False
return message.case
# if message contact has an open case, return that
existing_open = cls.get_open_for_contact_on(org, message.contact, timezone.now())
if existing_open:
existing_open.is_new = False
return existing_open
# suspend from groups, expire flows and archive messages
message.contact.prepare_for_case()
case = cls.objects.create(org=org, assignee=assignee, initial_message=message, contact=message.contact,
summary=summary)
case.is_new = True
case.labels.add(*list(message.labels.all())) # copy labels from message to new case
# attach message to this case
message.case = case
message.save(update_fields=('case',))
CaseAction.create(case, user, CaseAction.OPEN, assignee=assignee)
return case
def remove_expired_flows_from_active(apps, schema_editor):
r = get_redis_connection()
for key in r.keys('*:step_active_set:*'):
# make sure our flow run activity is removed
runs = FlowRun.objects.filter(pk__in=r.smembers(key),
is_active=False,
contact__is_test=False)
FlowRun.bulk_exit(runs, FlowRun.EXIT_TYPE_EXPIRED)
def remove_expired_flows_from_active(apps, schema_editor):
r = get_redis_connection()
for key in r.keys('*:step_active_set:*'):
# make sure our flow run activity is removed
FlowRun.do_expire_runs(
FlowRun.objects.filter(pk__in=r.smembers(key),
is_active=False,
contact__is_test=False))
def flush():
global KEY_WHITE_IP, KEY_WHITE_UA, KEY_BLACK_IP, KEY_BLACK_UA
global CACHE_WHITE_IP, CACHE_WHITE_UA, CACHE_BLACK_IP, CACHE_BLACK_UA
conn = get_redis_connection(settings.ANTI_SPAM_CACHE_REDIS_KEY)
CACHE_WHITE_IP = conn.smembers(KEY_WHITE_IP)
CACHE_WHITE_UA = conn.smembers(KEY_WHITE_UA)
CACHE_BLACK_IP = conn.smembers(KEY_BLACK_IP)
CACHE_BLACK_UA = conn.smembers(KEY_BLACK_UA)
def redis_check(request):
con = get_redis_connection('default')
hashs = {'key1':'kfs','dfas':'dasf'}
cache.set('key1',hashs)
tmp= cache.get('key1')
return HttpResponse(render_to_response('index.html',{'title':'redis','logs':[con,tmp]},context_instance=RequestContext(request)))
def generate_invoice_number(sequence_name=None):
"""
WARNING: This method changes the state in Redis!
"""
if sequence_name is None:
sequence_name = app_settings.INVOICE_NUMBER_SEQUENCE_NAME
conn = get_redis_connection()
return int(conn.incr(sequence_name))
def cron_update_index(sender, **kwargs):
index_redis = get_redis_connection('djrq')
index_count = int(index_redis.incr(settings.INDEX_NAME))
if index_count > settings.INDEX_COUNT + 1:
index_redis.set(settings.INDEX_NAME, 0)
index_queue = django_rq.get_queue(settings.INDEX_QUEUE)
if index_queue.count < 1:
index_queue.enqueue(warp_update_index)
def add_graphic(self):
graphic_r = get_redis_connection('graphic')
fdate = self.fcard.pub_date
self.nowdate = fdate
graphic_rid = str(self.fcard.mcard.id) + '_' + str(
self.fcard.carduser.id)
try:
grap = Graphic.objects.get(card=self.fcard.mcard,
fcard=self.fcard.mfcard)
grap_json = simplejson.loads(grap.graphic)
t_year = grap_json.get(str(fdate.year))
if t_year:
t_month = t_year.get(str(fdate.month))
if t_month:
t_day = t_month.get(str(fdate.day))
if t_day:
t_day.append(str(fdate.time())[:8])
else:
t_month[fdate.day] = [str(fdate.time())[:8]]
else:
t_year.update(
{fdate.month: {
fdate.day: [str(fdate.time())[:8]]
}})
else:
date_json = dict({
fdate.year: {
fdate.month: {
fdate.day: [str(fdate.time())[:8]]
}
}
})
grap_json.update(date_json)
grap.graphic = simplejson.dumps(grap_json)
grap.save(update_fields=['graphic'])
except Graphic.DoesNotExist:
grap_json = dict({
fdate.year: {
fdate.month: {
fdate.day: [str(fdate.time())[:8]]
}
}
})
grap = Graphic(graphic=simplejson.dumps(grap_json),
card=self.fcard.mcard,
fcard=self.fcard.mfcard)
grap.save()
self.fcard.graphic = grap
self.fcard.save(update_fields=['graphic'])
#update graphic redis
try:
nowdate_json = grap_json.get(str(fdate.year)).get(str(fdate.month))
except AttributeError:
nowdate_json = grap_json.get(fdate.year).get(fdate.month)
self.update_gredis(graphic_r, graphic_rid, nowdate_json, fdate)
def get_notifications(user):
r = get_redis_connection()
user_key = get_user_key(user)
notifications = r.hgetall(user_key).values()
notifications = [
json.loads(notification) for notification in notifications
]
notifications.sort(key=itemgetter('timestamp'))
return notifications
def get(self, request, *args, **kwargs):
auth_card = self.get_object()
graphic_r = get_redis_connection('graphic')
allg = graphic_r.hgetall(str(auth_card.id) + '_' + kwargs.get('user_pk'))
param = allg.get(kwargs.get('gyear')+'_'+kwargs.get("gmonth")+'_d')
if param:
param = simplejson.loads(param)
rparam = param.get(kwargs.get("gday")) if param else []
return HttpResponse(simplejson.dumps(rparam))
def add_tags(prefix, tagged_id, tags):
# тут просто добавляем сразу много тегов
r = get_redis_connection()
for t in tags:
if t.strip() == '':
continue
r.execute_command('ZINCRBY', key_all_tags, 1, t)
r.sadd(key(prefix, tagged_id), t)
r.sadd(tag_key(prefix, t), tagged_id)
def updatefavlist(user_id, card_id, action, target):
favlist_r = get_redis_connection('favlist')
fa_list = favlist_r.hget(str(user_id), card_id)
if fa_list == None or fa_list[0] != action :
if target != None:
action = action +'_'+target
favlist_r.hset(str(user_id), card_id, action)
else:
new_action = fa_list + '_'+target
favlist_r.hset(str(user_id), card_id, new_action)
def squash_contactgroupcounts():
"""
Squashes our ContactGroupCounts into single rows per ContactGroup
"""
r = get_redis_connection()
key = 'squash_channelcounts'
if not r.get(key):
with r.lock(key, timeout=900):
ContactGroupCount.squash_counts()
def get(self, request, *args, **kwargs):
auth_card = self.get_object()
graphic_r = get_redis_connection('graphic')
allg = graphic_r.hgetall(str(auth_card.id) + '_' + kwargs.get('user_pk'))
tmp = {}
year_p = re.compile('^\d{4}$')
for i in allg:
if re.match(year_p, i):
tmp.update({i:allg[i]})
return HttpResponse(simplejson.dumps(tmp))
def index(request):
redis = get_redis_connection('redis')
first_generate(redis)
incr_generate(redis)
code = get_invcode(redis)
res = {
'ret': 0,
'code': code
}
return HttpResponse(simplejson.dumps(res), mimetype="application/json; charset=utf-8")
def updatefavlist(user_id, card_id, action, target):
favlist_r = get_redis_connection('favlist')
fa_list = favlist_r.hget(str(user_id), card_id)
if fa_list == None or fa_list[0] != action:
if target != None:
action = action + '_' + target
favlist_r.hset(str(user_id), card_id, action)
else:
new_action = fa_list + '_' + target
favlist_r.hset(str(user_id), card_id, new_action)
def calculate_flow_stats_task(flow_id):
r = get_redis_connection()
flow = Flow.objects.get(pk=flow_id)
runs_started_cached = r.get(flow.get_stats_cache_key(FlowStatsCache.runs_started_count))
runs_started_cached = 0 if runs_started_cached is None else int(runs_started_cached)
runs_started = flow.runs.filter(contact__is_test=False).count()
if runs_started != runs_started_cached:
Flow.objects.get(pk=flow_id).do_calculate_flow_stats()
def check_messages_task():
"""
Checks to see if any of our aggregators have errored messages that need to be retried.
Also takes care of flipping Contacts from Failed to Normal and back based on their status.
"""
from django.utils import timezone
from .models import INCOMING, OUTGOING, PENDING, QUEUED, ERRORED, FAILED, WIRED, SENT, DELIVERED
from temba.contacts.models import NORMAL
from temba.orgs.models import Org
from temba.channels.tasks import send_msg_task
r = get_redis_connection()
# only do this if we aren't already running
key = 'check_messages_task'
if not r.get(key):
with r.lock(key, timeout=900):
now = timezone.now()
five_minutes_ago = now - timedelta(minutes=5)
# get any contacts that are currently normal that had a failed message in the past five minutes
for contact in Contact.objects.filter(msgs__created_on__gte=five_minutes_ago, msgs__direction=OUTGOING,
msgs__status=FAILED, status=NORMAL):
# if the last message from this contact is failed, then fail this contact
if contact.msgs.all().order_by('-created_on').first().status == FAILED:
contact.fail()
# get any contacts that are currently failed that had a normal message in the past five minutes
for contact in Contact.objects.filter(msgs__created_on__gte=five_minutes_ago, msgs__direction=OUTGOING,
msgs__status__in=[WIRED, SENT, DELIVERED], status=FAILED):
# if the last message from this contact is ok, then mark them as normal
if contact.msgs.all().order_by('-created_on').first().status in [WIRED, SENT, DELIVERED]:
contact.unfail()
# for any org that sent messages in the past five minutes, check for pending messages
for org in Org.objects.filter(msgs__created_on__gte=five_minutes_ago).distinct():
org.trigger_send()
# fire a few send msg tasks in case we dropped one somewhere during a restart
# (these will be no-ops if there is nothing to do)
send_msg_task.delay()
send_msg_task.delay()
handle_event_task.delay()
handle_event_task.delay()
# also check any incoming messages that are still pending somehow, reschedule them to be handled
unhandled_messages = Msg.objects.filter(direction=INCOMING, status=PENDING, created_on__lte=five_minutes_ago)
unhandled_messages = unhandled_messages.exclude(channel__org=None).exclude(contact__is_test=True)
unhandled_count = unhandled_messages.count()
if unhandled_count:
print "** Found %d unhandled messages" % unhandled_count
for msg in unhandled_messages:
msg.handle()
def collect_message_metrics_task():
"""
Collects message metrics and sends them to our analytics.
"""
from .models import INCOMING, OUTGOING, PENDING, QUEUED, ERRORED, INITIALIZING
from temba.utils import analytics
r = get_redis_connection()
# only do this if we aren't already running
key = 'collect_message_metrics'
if not r.get(key):
with r.lock(key, timeout=900):
# current # of queued messages (excluding Android)
count = Msg.objects.filter(direction=OUTGOING, status=QUEUED).exclude(channel=None).\
exclude(topup=None).exclude(channel__channel_type='A').exclude(next_attempt__gte=timezone.now()).count()
analytics.gauge('temba.current_outgoing_queued', count)
# current # of initializing messages (excluding Android)
count = Msg.objects.filter(
direction=OUTGOING,
status=INITIALIZING).exclude(channel=None).exclude(
topup=None).exclude(channel__channel_type='A').count()
analytics.gauge('temba.current_outgoing_initializing', count)
# current # of pending messages (excluding Android)
count = Msg.objects.filter(
direction=OUTGOING,
status=PENDING).exclude(channel=None).exclude(
topup=None).exclude(channel__channel_type='A').count()
analytics.gauge('temba.current_outgoing_pending', count)
# current # of errored messages (excluding Android)
count = Msg.objects.filter(
direction=OUTGOING,
status=ERRORED).exclude(channel=None).exclude(
topup=None).exclude(channel__channel_type='A').count()
analytics.gauge('temba.current_outgoing_errored', count)
# current # of android outgoing messages waiting to be sent
count = Msg.objects.filter(
direction=OUTGOING,
status__in=[PENDING, QUEUED],
channel__channel_type='A').exclude(channel=None).exclude(
topup=None).count()
analytics.gauge('temba.current_outgoing_android', count)
# current # of pending incoming messages that haven't yet been handled
count = Msg.objects.filter(
direction=INCOMING,
status=PENDING).exclude(channel=None).count()
analytics.gauge('temba.current_incoming_pending', count)
# stuff into redis when we last run, we do this as a canary as to whether our tasks are falling behind or not running
cache.set('last_cron', timezone.now())
def collect_message_metrics_task():
"""
Collects message metrics and sends them to our analytics.
"""
from .models import INCOMING, OUTGOING, DELIVERED, SENT, WIRED, FAILED, PENDING, QUEUED, ERRORED, INITIALIZING, HANDLED
import analytics
r = get_redis_connection()
# only do this if we aren't already running
key = 'collect_message_metrics'
if not r.get(key):
with r.lock(key, timeout=900):
# we use our hostname as our source so we can filter these for different brands
context = dict(source=settings.HOSTNAME)
# total # of delivered messages
count = Msg.objects.filter(direction=OUTGOING, status=DELIVERED).exclude(channel=None).exclude(topup=None).count()
analytics.track('System', 'temba.total_outgoing_delivered', properties=dict(value=count), context=context)
# total # of sent messages (this includes delivered and wired)
count = Msg.objects.filter(direction=OUTGOING, status__in=[DELIVERED, SENT, WIRED]).exclude(channel=None).exclude(topup=None).count()
analytics.track('System', 'temba.total_outgoing_sent', properties=dict(value=count), context=context)
# total # of failed messages
count = Msg.objects.filter(direction=OUTGOING, status=FAILED).exclude(channel=None).exclude(topup=None).count()
analytics.track('System', 'temba.total_outgoing_failed', properties=dict(value=count), context=context)
# current # of queued messages (excluding Android)
count = Msg.objects.filter(direction=OUTGOING, status=QUEUED).exclude(channel=None).exclude(topup=None).exclude(channel__channel_type='A').count()
analytics.track('System', 'temba.current_outgoing_queued', properties=dict(value=count), context=context)
# current # of initializing messages (excluding Android)
count = Msg.objects.filter(direction=OUTGOING, status=INITIALIZING).exclude(channel=None).exclude(topup=None).exclude(channel__channel_type='A').count()
analytics.track('System', 'temba.current_outgoing_initializing', properties=dict(value=count), context=context)
# current # of pending messages (excluding Android)
count = Msg.objects.filter(direction=OUTGOING, status=PENDING).exclude(channel=None).exclude(topup=None).exclude(channel__channel_type='A').count()
analytics.track('System', 'temba.current_outgoing_pending', properties=dict(value=count), context=context)
# current # of errored messages (excluding Android)
count = Msg.objects.filter(direction=OUTGOING, status=ERRORED).exclude(channel=None).exclude(topup=None).exclude(channel__channel_type='A').count()
analytics.track('System', 'temba.current_outgoing_errored', properties=dict(value=count), context=context)
# current # of android outgoing messages waiting to be sent
count = Msg.objects.filter(direction=OUTGOING, status__in=[PENDING, QUEUED], channel__channel_type='A').exclude(channel=None).exclude(topup=None).count()
analytics.track('System', 'temba.current_outgoing_android', properties=dict(value=count), context=context)
# current # of pending incoming messages that haven't yet been handled
count = Msg.objects.filter(direction=INCOMING, status=PENDING).exclude(channel=None).count()
analytics.track('System', 'temba.current_incoming_pending', properties=dict(value=count), context=context)
# stuff into redis when we last run, we do this as a canary as to whether our tasks are falling behind or not running
cache.set('last_cron', timezone.now())
