def active_pids(self, pids): stored_pids = self.get_by_key('pids') or set() for pid in stored_pids: if pid not in pids: LOG.debug('Purge dead uwsgi pid %s from pids list', pid) self.delete_by_key('pid:%s' % pid) self.set_by_key('pids', pids) # remove unneeded updates min_update = self._get_update_count() for pid in pids: n = self.get_by_key('pid:%s' % pid) if n: if n < min_update: min_update = n first_valid_update = self.get_by_key('first_valid_update') or 0 LOG.debug('Purge polled updates from %(first)s to %(min)s', {'first': first_valid_update, 'min': min_update}) for delete_id_set in utils.make_range(first_valid_update, min_update, BULK_DELETE_SIZE): if not self.memcached.delete_multi(delete_id_set, key_prefix=UPDATE_ID_PREFIX): LOG.critical('Failed to delete_multi from memcached') raise Exception('Failed to delete_multi from memcached') self.set_by_key('first_valid_update', min_update)
def _test_one_range(self, start, end, step): elements = set() for chunk in utils.make_range(start, end, step): for item in chunk: self.assertFalse(item in elements) elements.add(item) self.assertTrue(set(range(start, end)) == elements)
def get_update(self, pid): last_update = self.get_by_key('pid:%s' % pid) update_count = self._get_update_count() self.set_by_key('pid:%s' % pid, update_count) self._set_pids(pid) if not last_update: for i in self.get_all_records(): yield i else: for update_id_set in utils.make_range(last_update, update_count, BULK_READ_SIZE): update_set = self.memcached.get_multi( update_id_set, UPDATE_ID_PREFIX).values() for i in self.memcached.get_multi( update_set, RECORD_ID_PREFIX).values(): yield i
def get_all_records(self): for record_id_set in utils.make_range(0, self._get_record_count(), BULK_READ_SIZE): for i in self.memcached.get_multi( record_id_set, RECORD_ID_PREFIX).values(): yield i