def active_pids(self, pids):
stored_pids = self.get_by_key('pids') or set()
for pid in stored_pids:
if pid not in pids:
LOG.debug('Purge dead uwsgi pid %s from pids list', pid)
self.delete_by_key('pid:%s' % pid)
self.set_by_key('pids', pids)
# remove unneeded updates
min_update = self._get_update_count()
for pid in pids:
n = self.get_by_key('pid:%s' % pid)
if n:
if n < min_update:
min_update = n
first_valid_update = self.get_by_key('first_valid_update') or 0
LOG.debug('Purge polled updates from %(first)s to %(min)s', {
'first': first_valid_update,
'min': min_update
})
for delete_id_set in utils.make_range(first_valid_update, min_update,
BULK_DELETE_SIZE):
if not self.memcached.delete_multi(delete_id_set,
key_prefix=UPDATE_ID_PREFIX):
LOG.critical('Failed to delete_multi from memcached')
raise Exception('Failed to delete_multi from memcached')
self.set_by_key('first_valid_update', min_update)
def active_pids(self, pids):
stored_pids = self.get_by_key('pids') or set()
for pid in stored_pids:
if pid not in pids:
LOG.debug('Purge dead uwsgi pid %s from pids list', pid)
self.delete_by_key('pid:%s' % pid)
self.set_by_key('pids', pids)
# remove unneeded updates
min_update = self._get_update_count()
for pid in pids:
n = self.get_by_key('pid:%s' % pid)
if n:
if n < min_update:
min_update = n
first_valid_update = self.get_by_key('first_valid_update') or 0
LOG.debug('Purge polled updates from %(first)s to %(min)s',
{'first': first_valid_update, 'min': min_update})
for delete_id_set in utils.make_range(first_valid_update, min_update,
BULK_DELETE_SIZE):
if not self.memcached.delete_multi(delete_id_set,
key_prefix=UPDATE_ID_PREFIX):
LOG.critical('Failed to delete_multi from memcached')
raise Exception('Failed to delete_multi from memcached')
self.set_by_key('first_valid_update', min_update)
def export_data(memcached_inst, fd):
LOG.info('Exporting data from memcached')
for key in SINGLE_KEYS:
pickle.dump((key, memcached_inst.get(key)), fd)
for key in get_repo_keys(memcached_inst):
pickle.dump((key, memcached_inst.get(key)), fd)
for key in ARRAY_KEYS:
key_count = key + ':count'
count = memcached_inst.get(key_count) or 0
pickle.dump((key_count, memcached_inst.get(key_count)), fd)
key_prefix = key + ':'
for record_id_set in utils.make_range(0, count, BULK_READ_SIZE):
for k, v in six.iteritems(memcached_inst.get_multi(
record_id_set, key_prefix)):
pickle.dump((key_prefix + str(k), v), fd)
for user_seq in range(memcached_inst.get('user:count') or 0):
user = memcached_inst.get('user:%s' % user_seq)
if user:
if user.get('user_id'):
pickle.dump(('user:%s' % user['user_id'], user), fd)
if user.get('launchpad_id'):
pickle.dump(('user:%s' % user['launchpad_id'], user), fd)
for email in user.get('emails') or []:
pickle.dump(('user:%s' % email, user), fd)
def _test_one_range(self, start, end, step):
elements = set()
for chunk in utils.make_range(start, end, step):
for item in chunk:
self.assertFalse(item in elements)
elements.add(item)
self.assertTrue(set(range(start, end)) == elements)
def _test_one_range(self, start, end, step):
elements = set()
for chunk in utils.make_range(start, end, step):
for item in chunk:
self.assertFalse(item in elements)
elements.add(item)
self.assertTrue(set(range(start, end)) == elements)
def _test_one_range(self, start, end, step):
elements = set()
for chunk in utils.make_range(start, end, step):
for item in chunk:
self.assertNotIn(item, elements)
elements.add(item)
self.assertSetEqual(set(range(start, end)), elements)
def _test_one_range(self, start, end, step):
elements = set()
for chunk in utils.make_range(start, end, step):
for item in chunk:
self.assertNotIn(item, elements)
elements.add(item)
self.assertSetEqual(set(range(start, end)), elements)
def export_data(memcached_inst, fd):
LOG.info('Exporting data from memcached')
for key in SINGLE_KEYS:
pickle.dump((key, memcached_inst.get(key)), fd)
for key in get_repo_keys(memcached_inst):
pickle.dump((key, memcached_inst.get(key)), fd)
for key in ARRAY_KEYS:
key_count = key + ':count'
count = memcached_inst.get(key_count) or 0
pickle.dump((key_count, memcached_inst.get(key_count)), fd)
key_prefix = key + ':'
for record_id_set in utils.make_range(0, count + 1, BULK_READ_SIZE):
# memcache limits the size of returned data to specific yet unknown
# chunk size, the code should verify that all requested records are
# returned an be able to fall back to one-by-one retrieval
chunk = memcached_inst.get_multi(record_id_set, key_prefix)
if len(chunk) < len(record_id_set):
# retrieve one-by-one
for record_id in record_id_set:
key = key_prefix + str(record_id)
pickle.dump((key, memcached_inst.get(key)), fd)
else:
# dump the whole chunk
for k, v in six.iteritems(chunk):
pickle.dump((key_prefix + str(k), v), fd)
for user_seq in range((memcached_inst.get('user:count') or 0) + 1):
user = memcached_inst.get('user:%s' % user_seq)
if user:
if user.get('user_id'):
pickle.dump(
(('user:%s' % user['user_id']).encode('utf8'), user), fd)
if user.get('launchpad_id'):
pickle.dump(('user:%s' % user['launchpad_id'], user), fd)
for hostname, ids in user.get('gerrit_ids', {}).items():
for gerrit_id in ids:
pickle.dump(
('user:gerrit:%s:%s' % (hostname, gerrit_id), user),
fd)
if user.get('member_id'):
pickle.dump(('user:member:%s' % user['member_id'], user), fd)
for email in user.get('emails') or []:
pickle.dump((('user:%s' % email).encode('utf8'), user), fd)
LOG.info('Exporting data from memcached was completed')
def get_update(self, pid):
last_update = self.memcached.get("pid:%s" % pid)
update_count = self._get_update_count()
self.memcached.set("pid:%s" % pid, update_count)
self._set_pids(pid)
if not last_update:
for i in self.get_all_records():
yield i
else:
for update_id_set in utils.make_range(last_update, update_count, BULK_READ_SIZE):
update_set = self.memcached.get_multi(update_id_set, UPDATE_ID_PREFIX).values()
for i in self.memcached.get_multi(update_set, RECORD_ID_PREFIX).values():
yield i
def export_data(memcached_inst, fd):
LOG.info('Exporting data from memcached')
for key in SINGLE_KEYS:
pickle.dump((key, memcached_inst.get(key)), fd)
for key in get_repo_keys(memcached_inst):
pickle.dump((key, memcached_inst.get(key)), fd)
for key in ARRAY_KEYS:
key_count = key + ':count'
count = memcached_inst.get(key_count) or 0
pickle.dump((key_count, memcached_inst.get(key_count)), fd)
key_prefix = key + ':'
for record_id_set in utils.make_range(0, count + 1, BULK_READ_SIZE):
# memcache limits the size of returned data to specific yet unknown
# chunk size, the code should verify that all requested records are
# returned an be able to fall back to one-by-one retrieval
chunk = memcached_inst.get_multi(record_id_set, key_prefix)
if len(chunk) < len(record_id_set):
# retrieve one-by-one
for record_id in record_id_set:
key = key_prefix + str(record_id)
pickle.dump((key, memcached_inst.get(key)), fd)
else:
# dump the whole chunk
for k, v in six.iteritems(chunk):
pickle.dump((key_prefix + str(k), v), fd)
for user_seq in range((memcached_inst.get('user:count') or 0) + 1):
user = memcached_inst.get('user:%s' % user_seq)
if user:
if user.get('user_id'):
pickle.dump((('user:%s' % user['user_id']).encode('utf8'),
user), fd)
if user.get('launchpad_id'):
pickle.dump(('user:%s' % user['launchpad_id'], user), fd)
if user.get('gerrit_id'):
pickle.dump(('user:gerrit:%s' % user['gerrit_id'], user), fd)
if user.get('member_id'):
pickle.dump(('user:member:%s' % user['member_id'], user), fd)
for email in user.get('emails') or []:
pickle.dump((('user:%s' % email).encode('utf8'), user), fd)
def get_update(self, pid):
last_update = self.get_by_key('pid:%s' % pid)
update_count = self._get_update_count()
self.set_by_key('pid:%s' % pid, update_count)
self._set_pids(pid)
if not last_update:
for i in self.get_all_records():
yield i
else:
for update_id_set in utils.make_range(last_update, update_count,
BULK_READ_SIZE):
update_set = self.memcached.get_multi(
update_id_set, UPDATE_ID_PREFIX).values()
for i in self.memcached.get_multi(update_set,
RECORD_ID_PREFIX).values():
yield i
def active_pids(self, pids):
stored_pids = self.memcached.get("pids") or set()
for pid in stored_pids:
if pid not in pids:
self.memcached.delete("pid:%s" % pid)
self.memcached.set("pids", pids)
# remove unneeded updates
min_update = self._get_update_count()
for pid in pids:
n = self.memcached.get("pid:%s" % pid)
if n:
if n < min_update:
min_update = n
first_valid_update = self.memcached.get("first_valid_update") or 0
for delete_id_set in utils.make_range(first_valid_update, min_update, BULK_DELETE_SIZE):
if not self.memcached.delete_multi(delete_id_set, key_prefix=UPDATE_ID_PREFIX):
raise Exception("Failed to delete from memcache")
self.memcached.set("first_valid_update", min_update)
def active_pids(self, pids):
stored_pids = self.memcached.get('pids') or set()
for pid in stored_pids:
if pid not in pids:
self.memcached.delete('pid:%s' % pid)
self.memcached.set('pids', pids)
# remove unneeded updates
min_update = self._get_update_count()
for pid in pids:
n = self.memcached.get('pid:%s' % pid)
if n:
if n < min_update:
min_update = n
first_valid_update = self.memcached.get('first_valid_update') or 0
for delete_id_set in utils.make_range(first_valid_update, min_update,
BULK_DELETE_SIZE):
if not self.memcached.delete_multi(delete_id_set,
key_prefix=UPDATE_ID_PREFIX):
raise Exception('Failed to delete from memcache')
self.memcached.set('first_valid_update', min_update)
def get_all_records(self):
for record_id_set in utils.make_range(0, self._get_record_count(),
BULK_READ_SIZE):
for i in self.memcached.get_multi(record_id_set,
RECORD_ID_PREFIX).values():
yield i
def get_all_records(self):
for record_id_set in utils.make_range(0, self._get_record_count(),
BULK_READ_SIZE):
for i in self.memcached.get_multi(
record_id_set, RECORD_ID_PREFIX).values():
yield i