def get_percent_complete(self): ''' This method calculates how many sends/queues exist and makes a percentage If percentage is 100 it saves the dispatch as completed If not its cached for 15 seconds ''' if self.percent_complete == 100: return 100 else: key = ('dispatcher_%s_percent' % self.id) if redis.exists(key): return int(redis.get(key)) else: curr_sent_count = int(self.get_sent()) queued = int(self.queued_count if self.queued_count else self. get_queued()) processed = curr_sent_count + int(self.get_skipped()) percent = int( (float(processed) / queued) * 100) if queued > 0 else 0 # Note: self.queued_count is only set after queuing is done # percent needs to be >98 to allow old/dead jobs to run an hour or two later, data updates then too. if percent > 98 and self.queued_count: self.sent_count = curr_sent_count self.skipped_count = int(self.get_skipped()) self.percent_complete = 100 self.state = 10 self.save() self.expire_counts(86400) else: self.update(sent_count=curr_sent_count) redis.set(key, percent) redis.expire( key, 15) # cache current percent done for 15 seconds return percent
def test_get_percent_complete_cache(self): stack = helpers.create_stack() d = Dispatcher.find_by_id_anon(stack['dispatcher_id']) d.expire_counts() redis.set('dispatcher_%s_percent' % d.id, 15) self.assertEqual(d.get_percent_complete(), 15) redis.delete('dispatcher_%s_percent' % d.id)
def run_import(list_id): from liaison.models.list import List from liaison.lib.aws import get_contents_as_string from liaison.lib.extensions import redis_store as redis key = 'list_%s_import' % list_id list_ = List.find_by_id_anon(list_id) if list_: filename = list_.filename logger.info("\n filename: %s \n" % str(filename)) if "lists/" in filename: f = get_contents_as_string(filename) else: f = open(filename, 'rt') data = [] try: data = [ row for row in csv.DictReader( f.read().splitlines(), restkey='rk', restval='rv') ] list_.import_data = data result = list_.save() if result: redis.set(key, 'Upload Successful.') else: redis.set(key, 'List failed to save, the data may be corrupted.') except Exception, e: logger.info("Error: import_failure_list: %s: \n %s" % (e, traceback.format_exc())) redis.set(key, 'List failed to save, the data may be corrupted.') raise e finally:
def run_blacklist_import(account_id, filename): from liaison.models.blacklist import Blacklist from liaison.lib.aws import get_contents_as_string from liaison.lib.extensions import redis_store as redis key = 'blacklist_import_%s' % account_id if "Users/" in filename: f = open(filename, 'rt') else: f = get_contents_as_string(filename) try: data = [ row for row in csv.DictReader( f.read().splitlines(), restkey='rk', restval='rv') ] ok_key = True for row in data: if ok_key: spellings = ('email', 'Email', 'email_address', 'Email_Address', 'email address', 'Email Address', 'EmailAddress') intersect = set(spellings).intersection(row.keys()) intersect = intersect.pop() if intersect else None if intersect: email = row.get(intersect) reason = row.get('reason') detail = row.get('detail') Blacklist.insert(account_id, email, reason, detail) else: ok_key = False if ok_key: redis.set(key, 'Upload Successful.') except Exception, e: logger.info("Error: import_failure_blacklist: %s: \n %s" % (e, traceback.format_exc())) redis.set(key, 'Upload failed, the data may be corrupted.') raise e
def status(): """Check the database and cache, and report their status.""" services = {} code = 200 # DB if isinstance(Account.query.all(), list): services['db'] = 'ok' else: services['db'] = 'offline' code = 503 # Cache. Offline is ok on dev. To test use SimpleCache instead of NullCache. redis.set('status-chck', 'a-ok') redis.expire('status-chck', 2) if redis.get('status-chck') == 'a-ok': services['cache'] = 'ok' else: services['cache'] = 'offline' code = 503 services['status_code'] = code services['redis-online'] = check_redis() return jsonify(services), code
def set_cache(key, data, expiring=None): if check_redis(): response = redis.set(key, data) if expiring: redis.expire(key, expiring) return response