def _deindex_chunk(self, chunk):
rdir = RdirClient(self.conf, pool_manager=self.conscience.pool_manager)
url = chunk['url']
volume_id = url.split('/', 3)[2]
chunk_id = url.split('/', 3)[3]
rdir.chunk_delete(volume_id, self.container_id,
self.content_id, chunk_id)
class ChunkOperator(object):
"""
Execute maintenance operations on chunks.
"""
def __init__(self, conf, logger=None):
self.conf = conf
self.logger = logger or get_logger(conf)
self.rdir_client = RdirClient(conf, logger=self.logger)
self.content_factory = ContentFactory(conf, logger=self.logger)
def rebuild(self, container_id, content_id, chunk_id_or_pos,
rawx_id=None, try_chunk_delete=False, allow_same_rawx=True):
"""
Try to find the chunk in the metadata of the specified object,
then rebuild it.
"""
try:
content = self.content_factory.get(container_id, content_id)
except ContentNotFound:
raise OrphanChunk('Content not found: possible orphan chunk')
chunk_size = 0
chunk_pos = None
if len(chunk_id_or_pos) < 32:
chunk_pos = chunk_id_or_pos
chunk_id = None
metapos = int(chunk_pos.split('.', 1)[0])
chunk_size = content.chunks.filter(metapos=metapos).all()[0].size
else:
if '/' in chunk_id_or_pos:
chunk_id = chunk_id_or_pos.rsplit('/', 1)[-1]
else:
chunk_id = chunk_id_or_pos
chunk = content.chunks.filter(id=chunk_id).one()
if chunk is None:
raise OrphanChunk(
'Chunk not found in content: possible orphan chunk')
elif rawx_id and chunk.host != rawx_id:
raise ValueError('Chunk does not belong to this rawx')
chunk_size = chunk.size
content.rebuild_chunk(
chunk_id, allow_same_rawx=allow_same_rawx,
chunk_pos=chunk_pos)
if try_chunk_delete:
try:
content.blob_client.chunk_delete(chunk.url)
self.logger.info("Chunk %s deleted", chunk.url)
except NotFound as exc:
self.logger.debug("Chunk %s: %s", chunk.url, exc)
# This call does not raise exception if chunk is not referenced
if chunk_id is not None:
self.rdir_client.chunk_delete(
chunk.host, container_id, content_id, chunk_id)
return chunk_size
class Harasser(object):
def __init__(self, ns, max_containers=256, max_contents=256):
conf = {'namespace': ns}
self.cs = ConscienceClient(conf)
self.rdir = RdirClient(conf)
self.rawx_list = [x['addr'] for x in self.cs.all_services('rawx')]
self.sent = set()
self.max_containers = max_containers
self.max_contents = max_contents
def harass_put(self, loops=None):
if loops is None:
loops = random.randint(1000, 2000)
print "Pushing %d fake chunks" % loops
loop = loops
count_start_container = random.randrange(2**20)
count_start_content = random.randrange(2**20)
start = time.time()
nb_rawx = len(self.rawx_list)
while loop > 0:
args = {'mtime': int(start)}
# vol_id = random.choice(self.rawx_list)
# container_id = "%064X" % (random.randrange(self.max_containers))
# content_id = "%032X" % (random.randrange(self.max_contents))
vol_id = self.rawx_list[loop % nb_rawx]
container_id = "%064X" % (loop + count_start_container)
content_id = "%032X" % (loop + count_start_content)
chunk_id = "http://%s/%064X" \
% (vol_id, random.randrange(2**128))
self.rdir.chunk_push(
vol_id, container_id, content_id, chunk_id, **args)
self.sent.add((vol_id, container_id, content_id, chunk_id))
loop -= 1
end = time.time()
print "%d pushed in %.3fs, %d req/s" \
% (loops, end-start, loops/(end-start))
def harass_del(self, min_loops=0):
min_loops = min(min_loops, len(self.sent))
loops = random.randint(min_loops, len(self.sent))
print "Removing %d fake chunks" % loops
loop = loops
start = time.time()
while loop > 0:
args = self.sent.pop()
self.rdir.chunk_delete(*args)
loop -= 1
end = time.time()
print "%d removed in %.3fs, %d req/s" \
% (loops, end-start, loops/(end-start))
def __call__(self):
try:
while True:
self.harass_put()
self.harass_del()
except KeyboardInterrupt:
print "Cleaning..."
self.harass_del(len(self.sent))
class BlobRebuilderWorker(object):
def __init__(self,
conf,
logger,
volume,
input_file=None,
try_chunk_delete=False,
beanstalkd_addr=None):
self.conf = conf
self.logger = logger or get_logger(conf)
self.volume = volume
self.run_time = 0
self.passes = 0
self.errors = 0
self.last_reported = 0
self.chunks_run_time = 0
self.bytes_running_time = 0
self.bytes_processed = 0
self.total_bytes_processed = 0
self.total_chunks_processed = 0
self.dry_run = true_value(conf.get('dry_run', False))
self.report_interval = int_value(conf.get('report_interval'), 3600)
self.max_chunks_per_second = int_value(conf.get('chunks_per_second'),
30)
self.max_bytes_per_second = int_value(conf.get('bytes_per_second'),
10000000)
self.rdir_fetch_limit = int_value(conf.get('rdir_fetch_limit'), 100)
self.allow_same_rawx = true_value(conf.get('allow_same_rawx'))
self.input_file = input_file
self.rdir_client = RdirClient(conf, logger=self.logger)
self.content_factory = ContentFactory(conf)
self.try_chunk_delete = try_chunk_delete
self.beanstalkd_addr = beanstalkd_addr
self.beanstalkd_tube = conf.get('beanstalkd_tube', 'rebuild')
self.beanstalk = None
def _fetch_chunks_from_event(self, job_id, data):
env = json.loads(data)
for chunk_pos in env['data']['missing_chunks']:
yield [
env['url']['id'], env['url']['content'],
str(chunk_pos), None
]
def _connect_to_beanstalk(self):
self.beanstalk = Beanstalk.from_url(self.beanstalkd_addr)
self.beanstalk.use(self.beanstalkd_tube)
self.beanstalk.watch(self.beanstalkd_tube)
def _handle_beanstalk_event(self, conn_error):
try:
job_id, data = self.beanstalk.reserve()
if conn_error:
self.logger.warn("beanstalk reconnected")
except ConnectionError:
if not conn_error:
self.logger.warn("beanstalk connection error")
raise
try:
for chunk in self._fetch_chunks_from_event(job_id, data):
yield chunk
self.beanstalk.delete(job_id)
except Exception:
self.logger.exception("handling event %s (bury)", job_id)
self.beanstalk.bury(job_id)
def _fetch_chunks_from_beanstalk(self):
conn_error = False
while 1:
try:
self._connect_to_beanstalk()
for chunk in self._handle_beanstalk_event(conn_error):
conn_error = False
yield chunk
except ConnectionError:
conn_error = True
time.sleep(1.0)
def _fetch_chunks_from_file(self):
with open(self.input_file, 'r') as ifile:
for line in ifile:
stripped = line.strip()
if stripped and not stripped.startswith('#'):
yield stripped.split('|', 3)[:3] + [None]
def _fetch_chunks(self):
if self.input_file:
return self._fetch_chunks_from_file()
elif self.beanstalkd_addr:
return self._fetch_chunks_from_beanstalk()
else:
return self.rdir_client.chunk_fetch(self.volume,
limit=self.rdir_fetch_limit,
rebuild=True)
def rebuilder_pass_with_lock(self):
self.rdir_client.admin_lock(self.volume,
"rebuilder on %s" % gethostname())
try:
self.rebuilder_pass()
finally:
self.rdir_client.admin_unlock(self.volume)
def rebuilder_pass(self):
start_time = report_time = time.time()
rebuilder_time = 0
chunks = self._fetch_chunks()
for cid, content_id, chunk_id_or_pos, _ in chunks:
loop_time = time.time()
if self.dry_run:
self.dryrun_chunk_rebuild(cid, content_id, chunk_id_or_pos)
else:
self.safe_chunk_rebuild(cid, content_id, chunk_id_or_pos)
self.chunks_run_time = ratelimit(self.chunks_run_time,
self.max_chunks_per_second)
self.total_chunks_processed += 1
now = time.time()
if now - self.last_reported >= self.report_interval:
self.logger.info(
'RUN %(volume)s '
'started=%(start_time)s '
'passes=%(passes)d '
'errors=%(errors)d '
'chunks=%(nb_chunks)d %(c_rate).2f/s '
'bytes=%(nb_bytes)d %(b_rate).2fB/s '
'elapsed=%(total).2f '
'(rebuilder: %(success_rate).2f%%)' % {
'volume':
self.volume,
'start_time':
datetime.fromtimestamp(int(report_time)).isoformat(),
'passes':
self.passes,
'errors':
self.errors,
'nb_chunks':
self.total_chunks_processed,
'nb_bytes':
self.total_bytes_processed,
'c_rate':
self.passes / (now - report_time),
'b_rate':
self.bytes_processed / (now - report_time),
'total': (now - start_time),
'rebuilder_time':
rebuilder_time,
'success_rate':
100 * ((self.total_chunks_processed - self.errors) /
float(self.total_chunks_processed))
})
report_time = now
self.passes = 0
self.bytes_processed = 0
self.last_reported = now
rebuilder_time += (now - loop_time)
end_time = time.time()
elapsed = (end_time - start_time) or 0.000001
self.logger.info(
'DONE %(volume)s '
'started=%(start_time)s '
'ended=%(end_time)s '
'passes=%(passes)d '
'elapsed=%(elapsed).02f '
'errors=%(errors)d '
'chunks=%(nb_chunks)d %(c_rate).2f/s '
'bytes=%(nb_bytes)d %(b_rate).2fB/s '
'elapsed=%(rebuilder_time).2f '
'(rebuilder: %(success_rate).2f%%)' % {
'volume':
self.volume,
'start_time':
datetime.fromtimestamp(int(start_time)).isoformat(),
'end_time':
datetime.fromtimestamp(int(end_time)).isoformat(),
'passes':
self.passes,
'elapsed':
elapsed,
'errors':
self.errors,
'nb_chunks':
self.total_chunks_processed,
'nb_bytes':
self.total_bytes_processed,
'c_rate':
self.total_chunks_processed / elapsed,
'b_rate':
self.total_bytes_processed / elapsed,
'rebuilder_time':
rebuilder_time,
'success_rate':
100 * ((self.total_chunks_processed - self.errors) /
float(self.total_chunks_processed or 1))
})
def dryrun_chunk_rebuild(self, container_id, content_id, chunk_id_or_pos):
self.logger.info(
"[dryrun] Rebuilding "
"container %s, content %s, chunk %s", container_id, content_id,
chunk_id_or_pos)
self.passes += 1
def safe_chunk_rebuild(self, container_id, content_id, chunk_id_or_pos):
try:
self.chunk_rebuild(container_id, content_id, chunk_id_or_pos)
except Exception as e:
self.errors += 1
self.logger.error('ERROR while rebuilding chunk %s|%s|%s: %s',
container_id, content_id, chunk_id_or_pos, e)
self.passes += 1
def chunk_rebuild(self, container_id, content_id, chunk_id_or_pos):
self.logger.info('Rebuilding (container %s, content %s, chunk %s)',
container_id, content_id, chunk_id_or_pos)
try:
content = self.content_factory.get(container_id, content_id)
except ContentNotFound:
raise OrphanChunk('Content not found: possible orphan chunk')
chunk_size = 0
chunk_pos = None
if len(chunk_id_or_pos) < 32:
chunk_pos = chunk_id_or_pos
chunk_id = None
metapos = int(chunk_pos.split('.', 1)[0])
chunk_size = content.chunks.filter(metapos=metapos).all()[0].size
else:
if '/' in chunk_id_or_pos:
chunk_id = chunk_id_or_pos.rsplit('/', 1)[-1]
else:
chunk_id = chunk_id_or_pos
chunk = content.chunks.filter(id=chunk_id).one()
if chunk is None:
raise OrphanChunk(("Chunk not found in content:"
"possible orphan chunk"))
elif self.volume and chunk.host != self.volume:
raise ValueError("Chunk does not belong to this volume")
chunk_size = chunk.size
content.rebuild_chunk(chunk_id,
allow_same_rawx=self.allow_same_rawx,
chunk_pos=chunk_pos)
if self.try_chunk_delete:
try:
content.blob_client.chunk_delete(chunk.url)
self.logger.info("Chunk %s deleted", chunk.url)
except NotFound as exc:
self.logger.debug("Chunk %s: %s", chunk.url, exc)
# This call does not raise exception if chunk is not referenced
if chunk_id is not None:
self.rdir_client.chunk_delete(chunk.host, container_id, content_id,
chunk_id)
self.bytes_processed += chunk_size
self.total_bytes_processed += chunk_size
class BlobRebuilderWorker(RebuilderWorker):
def __init__(self, conf, logger, volume, try_chunk_delete=False, **kwargs):
super(BlobRebuilderWorker, self).__init__(conf, logger, **kwargs)
self.volume = volume
self.bytes_processed = 0
self.total_bytes_processed = 0
self.dry_run = true_value(conf.get('dry_run', False))
self.allow_same_rawx = true_value(conf.get('allow_same_rawx'))
self.rdir_client = RdirClient(conf, logger=self.logger)
self.content_factory = ContentFactory(conf, logger=self.logger)
self.try_chunk_delete = try_chunk_delete
def _rebuild_one(self, chunk, **kwargs):
cid, content_id, chunk_id_or_pos, _ = chunk
if self.dry_run:
self.dryrun_chunk_rebuild(cid, content_id, chunk_id_or_pos)
else:
self.safe_chunk_rebuild(cid, content_id, chunk_id_or_pos)
def _get_report(self, num, start_time, end_time, total_time, report_time,
**kwargs):
return ('RUN %(volume)s '
'worker=%(num)d '
'started=%(start_time)s '
'passes=%(passes)d '
'errors=%(errors)d '
'chunks=%(nb_chunks)d %(c_rate).2f/s '
'bytes=%(nb_bytes)d %(b_rate).2fB/s '
'waiting_time=%(waiting_time).2f '
'rebuilder_time=%(rebuilder_time).2f '
'total_time=%(total_time).2f '
'(rebuilder: %(success_rate).2f%%)' % {
'volume':
self.volume,
'num':
num,
'start_time':
datetime.fromtimestamp(int(report_time)).isoformat(),
'passes':
self.passes,
'errors':
self.errors,
'nb_chunks':
self.total_items_processed,
'nb_bytes':
self.total_bytes_processed,
'c_rate':
self.passes / (end_time - report_time),
'b_rate':
self.bytes_processed / (end_time - report_time),
'waiting_time':
self.waiting_time,
'rebuilder_time':
self.rebuilder_time,
'total_time': (end_time - start_time),
'success_rate':
100 * ((self.total_items_processed - self.errors) /
float(self.total_items_processed))
})
def dryrun_chunk_rebuild(self, container_id, content_id, chunk_id_or_pos):
self.logger.info(
"[dryrun] Rebuilding "
"container %s, content %s, chunk %s", container_id, content_id,
chunk_id_or_pos)
self.passes += 1
def safe_chunk_rebuild(self, container_id, content_id, chunk_id_or_pos):
try:
self.chunk_rebuild(container_id, content_id, chunk_id_or_pos)
except Exception as e:
self.errors += 1
self.logger.error('ERROR while rebuilding chunk %s|%s|%s: %s',
container_id, content_id, chunk_id_or_pos, e)
self.passes += 1
def chunk_rebuild(self, container_id, content_id, chunk_id_or_pos):
self.logger.info('Rebuilding (container %s, content %s, chunk %s)',
container_id, content_id, chunk_id_or_pos)
try:
content = self.content_factory.get(container_id, content_id)
except ContentNotFound:
raise OrphanChunk('Content not found: possible orphan chunk')
chunk_size = 0
chunk_pos = None
if len(chunk_id_or_pos) < 32:
chunk_pos = chunk_id_or_pos
chunk_id = None
metapos = int(chunk_pos.split('.', 1)[0])
chunk_size = content.chunks.filter(metapos=metapos).all()[0].size
else:
if '/' in chunk_id_or_pos:
chunk_id = chunk_id_or_pos.rsplit('/', 1)[-1]
else:
chunk_id = chunk_id_or_pos
chunk = content.chunks.filter(id=chunk_id).one()
if chunk is None:
raise OrphanChunk(("Chunk not found in content:"
'possible orphan chunk'))
elif self.volume and chunk.host != self.volume:
raise ValueError("Chunk does not belong to this volume")
chunk_size = chunk.size
content.rebuild_chunk(chunk_id,
allow_same_rawx=self.allow_same_rawx,
chunk_pos=chunk_pos)
if self.try_chunk_delete:
try:
content.blob_client.chunk_delete(chunk.url)
self.logger.info("Chunk %s deleted", chunk.url)
except NotFound as exc:
self.logger.debug("Chunk %s: %s", chunk.url, exc)
# This call does not raise exception if chunk is not referenced
if chunk_id is not None:
self.rdir_client.chunk_delete(chunk.host, container_id, content_id,
chunk_id)
self.bytes_processed += chunk_size
self.total_bytes_processed += chunk_size
class EventWorker(object):
def __init__(self, conf, name, context, **kwargs):
self.conf = conf
self.name = name
verbose = kwargs.pop('verbose', False)
self.logger = get_logger(self.conf, verbose=verbose)
self.init_zmq(context)
self.cs = ConscienceClient(self.conf)
self.rdir = RdirClient(self.conf)
self._acct_addr = None
self.acct_update = 0
self.acct_refresh_interval = int_value(
conf.get('acct_refresh_interval'), 60
)
self.acct_update = true_value(
conf.get('acct_update', True))
self.session = requests.Session()
self.failed = False
def start(self):
self.logger.info('worker "%s" starting', self.name)
self.running = True
self.run()
def stop(self):
self.logger.info('worker "%s" stopping', self.name)
self.running = False
def init_zmq(self, context):
socket = context.socket(zmq.REP)
socket.connect('inproc://event-front')
self.socket = socket
def safe_ack(self, msg):
try:
self.socket.send_multipart(msg)
except Exception:
self.logger.warn('Unable to ack event')
def run(self):
try:
while self.running:
msg = self.socket.recv_multipart()
self.logger.debug("msg received: %s" % msg)
event = decode_msg(msg)
success = self.process_event(event)
f = "0" if success else ""
self.safe_ack([msg[0], f])
except Exception as e:
self.logger.warn('ERROR in worker "%s"', e)
self.failed = True
raise e
finally:
self.logger.info('worker "%s" stopped', self.name)
def process_event(self, event):
handler = self.get_handler(event)
if not handler:
self.logger.warn("No handler found")
# mark as success
return True
success = True
try:
handler(event)
except Exception:
success = False
finally:
return success
def get_handler(self, event):
event_type = event.get('event')
if not event_type:
return None
if event_type == EventType.CONTAINER_PUT:
return self.handle_container_put
elif event_type == EventType.CONTAINER_DESTROY:
return self.handle_container_destroy
elif event_type == EventType.CONTAINER_UPDATE:
return self.handle_container_update
elif event_type == EventType.OBJECT_PUT:
return self.handle_object_put
elif event_type == EventType.OBJECT_DELETE:
return self.handle_object_delete
elif event_type == EventType.REFERENCE_UPDATE:
return self.handle_reference_update
elif event_type == EventType.CHUNK_PUT:
return self.handle_chunk_put
elif event_type == EventType.CHUNK_DELETE:
return self.handle_chunk_delete
elif event_type == EventType.PING:
return self.handle_ping
else:
return None
@property
def acct_addr(self):
if not self._acct_addr or self.acct_refresh():
try:
acct_instance = self.cs.next_instance(ACCOUNT_SERVICE)
self._acct_addr = acct_instance.get('addr')
self.acct_update = time.time()
except Exception:
self.logger.warn('Unable to find account instance')
return self._acct_addr
def acct_refresh(self):
return (time.time() - self.acct_update) > self.acct_refresh_interval
def handle_container_put(self, event):
"""
Handle container creation.
:param event:
"""
self.logger.debug('worker "%s" handle container put', self.name)
if not self.acct_update:
return
uri = 'http://%s/v1.0/account/container/update' % self.acct_addr
mtime = event.get('when')
data = event.get('data')
name = data.get('url').get('user')
account = data.get('url').get('account')
event = {'mtime': mtime, 'name': name}
self.session.post(uri, params={'id': account}, data=json.dumps(event))
def handle_container_update(self, event):
"""
Handle container update.
:param event:
"""
self.logger.debug('worker "%s" handle container update', self.name)
if not self.acct_update:
return
uri = 'http://%s/v1.0/account/container/update' % self.acct_addr
mtime = event.get('when')
data = event.get('data')
name = event.get('url').get('user')
account = event.get('url').get('account')
bytes_count = data.get('bytes-count', 0)
object_count = data.get('object-count', 0)
event = {
'mtime': mtime,
'name': name,
'bytes': bytes_count,
'objects': object_count
}
self.session.post(uri, params={'id': account}, data=json.dumps(event))
def handle_container_destroy(self, event):
"""
Handle container destroy.
:param event:
"""
self.logger.debug('worker "%s" handle container destroy', self.name)
if not self.acct_update:
return
uri = 'http://%s/v1.0/account/container/update' % self.acct_addr
dtime = event.get('when')
data = event.get('data')
name = data.get('url').get('user')
account = data.get('url').get('account')
event = {'dtime': dtime, 'name': name}
self.session.post(uri, params={'id': account}, data=json.dumps(event))
def handle_object_delete(self, event):
"""
Handle object deletion.
Delete the chunks of the object.
:param event:
"""
self.logger.debug('worker "%s" handle object delete', self.name)
pile = GreenPile(PARALLEL_CHUNKS_DELETE)
chunks = []
for item in event.get('data'):
if item.get('type') == 'chunks':
chunks.append(item)
if not len(chunks):
self.logger.warn('No chunks found in event data')
return
def delete_chunk(chunk):
resp = None
try:
with Timeout(CHUNK_TIMEOUT):
resp = self.session.delete(chunk['id'])
except (Exception, Timeout) as e:
self.logger.exception(e)
return resp
for chunk in chunks:
pile.spawn(delete_chunk, chunk)
resps = [resp for resp in pile if resp]
for resp in resps:
if resp.status_code == 204:
self.logger.info('deleted chunk %s' % resp.url)
else:
self.logger.warn('failed to delete chunk %s' % resp.url)
def handle_object_put(self, event):
"""
Handle object creation.
TODO
:param event:
"""
self.logger.debug('worker "%s" handle object put', self.name)
def handle_reference_update(self, event):
"""
Handle reference update.
TODO
:param event
"""
self.logger.debug('worker "%s" handle reference update', self.name)
def handle_chunk_put(self, event):
"""
Handle chunk creation.
:param event
"""
self.logger.debug('worker "%s" handle chunk creation', self.name)
when = event.get('when')
data = event.get('data')
volume_id = data.get('volume_id')
del data['volume_id']
container_id = data.get('container_id')
del data['container_id']
content_id = data.get('content_id')
del data['content_id']
chunk_id = data.get('chunk_id')
del data['chunk_id']
data['mtime'] = when
self.rdir.chunk_push(volume_id, container_id, content_id, chunk_id,
**data)
def handle_chunk_delete(self, event):
"""
Handle chunk deletion.
:param event
"""
self.logger.debug('worker "%s" handle chunk deletion', self.name)
data = event.get('data')
volume_id = data.get('volume_id')
container_id = data.get('container_id')
content_id = data.get('content_id')
chunk_id = data.get('chunk_id')
self.rdir.chunk_delete(volume_id, container_id, content_id, chunk_id)
def handle_ping(self, event):
"""
Handle ping
:param event
"""
self.logger.debug('worker "%s" handle ping', self.name)
class BlobRebuilderWorker(object):
def __init__(self, conf, logger, volume):
self.conf = conf
self.logger = logger or get_logger(conf)
self.volume = volume
self.run_time = 0
self.passes = 0
self.errors = 0
self.last_reported = 0
self.chunks_run_time = 0
self.bytes_running_time = 0
self.bytes_processed = 0
self.total_bytes_processed = 0
self.total_chunks_processed = 0
self.dry_run = true_value(conf.get('dry_run', False))
self.report_interval = int_value(conf.get('report_interval'), 3600)
self.max_chunks_per_second = int_value(conf.get('chunks_per_second'),
30)
self.max_bytes_per_second = int_value(conf.get('bytes_per_second'),
10000000)
self.rdir_fetch_limit = int_value(conf.get('rdir_fetch_limit'), 100)
self.allow_same_rawx = true_value(conf.get('allow_same_rawx'))
self.rdir_client = RdirClient(conf)
self.content_factory = ContentFactory(conf)
def rebuilder_pass_with_lock(self):
self.rdir_client.admin_lock(self.volume,
"rebuilder on %s" % gethostname())
try:
self.rebuilder_pass()
finally:
self.rdir_client.admin_unlock(self.volume)
def rebuilder_pass(self):
start_time = report_time = time.time()
total_errors = 0
rebuilder_time = 0
chunks = self.rdir_client.chunk_fetch(self.volume,
limit=self.rdir_fetch_limit,
rebuild=True)
for container_id, content_id, chunk_id, data in chunks:
loop_time = time.time()
if self.dry_run:
self.dryrun_chunk_rebuild(container_id, content_id, chunk_id)
else:
self.safe_chunk_rebuild(container_id, content_id, chunk_id)
self.chunks_run_time = ratelimit(self.chunks_run_time,
self.max_chunks_per_second)
self.total_chunks_processed += 1
now = time.time()
if now - self.last_reported >= self.report_interval:
self.logger.info(
'RUN %(volume)s '
'started=%(start_time)s '
'passes=%(passes)d '
'errors=%(errors)d '
'chunks=%(nb_chunks)d %(c_rate).2f/s '
'bytes=%(nb_bytes)d %(b_rate).2fB/s '
'elapsed=%(total).2f '
'(rebuilder: %(rebuilder_rate).2f%%)' % {
'volume':
self.volume,
'start_time':
datetime.fromtimestamp(int(report_time)).isoformat(),
'passes':
self.passes,
'errors':
self.errors,
'nb_chunks':
self.total_chunks_processed,
'nb_bytes':
self.total_bytes_processed,
'c_rate':
self.passes / (now - report_time),
'b_rate':
self.bytes_processed / (now - report_time),
'total': (now - start_time),
'rebuilder_time':
rebuilder_time,
'rebuilder_rate':
100.0 * rebuilder_time / float(now - start_time)
})
report_time = now
total_errors += self.errors
self.passes = 0
self.bytes_processed = 0
self.last_reported = now
rebuilder_time += (now - loop_time)
end_time = time.time()
elapsed = (end_time - start_time) or 0.000001
self.logger.info(
'DONE %(volume)s '
'started=%(start_time)s '
'ended=%(end_time)s '
'elapsed=%(elapsed).02f '
'errors=%(errors)d '
'chunks=%(nb_chunks)d %(c_rate).2f/s '
'bytes=%(nb_bytes)d %(b_rate).2fB/s '
'elapsed=%(rebuilder_time).2f '
'(rebuilder: %(rebuilder_rate).2f%%)' % {
'volume': self.volume,
'start_time': datetime.fromtimestamp(
int(start_time)).isoformat(),
'end_time': datetime.fromtimestamp(int(end_time)).isoformat(),
'elapsed': elapsed,
'errors': total_errors + self.errors,
'nb_chunks': self.total_chunks_processed,
'nb_bytes': self.total_bytes_processed,
'c_rate': self.total_chunks_processed / elapsed,
'b_rate': self.total_bytes_processed / elapsed,
'rebuilder_time': rebuilder_time,
'rebuilder_rate': 100.0 * rebuilder_time / float(elapsed)
})
def dryrun_chunk_rebuild(self, container_id, content_id, chunk_id):
self.logger.info(
"[dryrun] Rebuilding "
"container %s, content %s, chunk %s", container_id, content_id,
chunk_id)
self.passes += 1
def safe_chunk_rebuild(self, container_id, content_id, chunk_id):
try:
self.chunk_rebuild(container_id, content_id, chunk_id)
except Exception as e:
self.errors += 1
self.logger.error('ERROR while rebuilding chunk %s|%s|%s) : %s',
container_id, content_id, chunk_id, e)
self.passes += 1
def chunk_rebuild(self, container_id, content_id, chunk_id):
self.logger.info('Rebuilding (container %s, content %s, chunk %s)',
container_id, content_id, chunk_id)
try:
content = self.content_factory.get(container_id, content_id)
except ContentNotFound:
raise exc.OrphanChunk('Content not found')
chunk = content.chunks.filter(id=chunk_id).one()
if chunk is None:
raise OrphanChunk("Chunk not found in content")
chunk_size = chunk.size
content.rebuild_chunk(chunk_id, allow_same_rawx=self.allow_same_rawx)
self.rdir_client.chunk_delete(self.volume, container_id, content_id,
chunk_id)
self.bytes_processed += chunk_size
self.total_bytes_processed += chunk_size
class ChunkOperator(object):
"""
Execute maintenance operations on chunks.
"""
def __init__(self, conf, logger=None):
self.conf = conf
self.logger = logger or get_logger(conf)
self.rdir_client = RdirClient(conf, logger=self.logger)
self.content_factory = ContentFactory(conf, logger=self.logger)
def rebuild(self,
container_id,
content_id,
chunk_id_or_pos,
rawx_id=None,
try_chunk_delete=False,
allow_frozen_container=True,
allow_same_rawx=True):
"""
Try to find the chunk in the metadata of the specified object,
then rebuild it.
"""
try:
content = self.content_factory.get(container_id, content_id)
except ContentNotFound:
raise OrphanChunk('Content not found: possible orphan chunk')
chunk_pos = None
if looks_like_chunk_position(chunk_id_or_pos):
chunk_pos = chunk_id_or_pos
chunk_id = None
else:
if '/' in chunk_id_or_pos:
parsed = urlparse(chunk_id_or_pos)
chunk_id = parsed.path.lstrip('/')
rawx_id = parsed.netloc
else:
chunk_id = chunk_id_or_pos
candidates = content.chunks.filter(id=chunk_id)
# FIXME(FVE): if for some reason the chunks have been registered
# with an IP address and port instead of an ID, this won't work.
if rawx_id:
candidates = candidates.filter(host=rawx_id)
chunk = candidates.one()
if chunk is None:
raise OrphanChunk(
'Chunk not found in content: possible orphan chunk: ' +
'%s' % (candidates.all(), ))
elif rawx_id and chunk.host != rawx_id:
raise ValueError('Chunk does not belong to this rawx')
rebuilt_bytes = content.rebuild_chunk(
chunk_id,
service_id=rawx_id,
allow_frozen_container=allow_frozen_container,
allow_same_rawx=allow_same_rawx,
chunk_pos=chunk_pos)
if try_chunk_delete:
try:
content.blob_client.chunk_delete(chunk.url)
self.logger.info("Old chunk %s deleted", chunk.url)
except Exception as exc:
self.logger.warn('Failed to delete old chunk %s: %s',
chunk.url, exc)
# This call does not raise exception if chunk is not referenced
if chunk_id is not None:
try:
self.rdir_client.chunk_delete(chunk.host, container_id,
content_id, chunk_id)
except Exception as exc:
self.logger.warn(
'Failed to delete chunk entry (%s) from the rdir (%s): %s',
chunk_id, chunk.host, exc)
return rebuilt_bytes
class Harasser(object):
def __init__(self, ns, max_containers=256, max_contents=256):
conf = {'namespace': ns}
self.cs = ConscienceClient(conf)
self.rdir = RdirClient(conf)
self.rawx_list = [x['addr'] for x in self.cs.all_services('rawx')]
self.sent = set()
self.max_containers = max_containers
self.max_contents = max_contents
self.pushed_count = 0
self.pushed_time = 0
self.removed_count = 0
self.removed_time = 0
def harass_put(self, loops=None):
if loops is None:
loops = random.randint(1000, 2000)
print("Pushing %d fake chunks" % loops)
loop = loops
count_start_container = random.randrange(2**20)
count_start_content = random.randrange(2**20)
start = time.time()
nb_rawx = len(self.rawx_list)
while loop > 0:
args = {'mtime': int(start)}
# vol_id = random.choice(self.rawx_list)
# container_id = "%064X" % (random.randrange(self.max_containers))
# content_id = "%032X" % (random.randrange(self.max_contents))
vol_id = self.rawx_list[loop % nb_rawx]
container_id = "%064X" % (loop + count_start_container)
content_id = "%032X" % (loop + count_start_content)
chunk_id = "http://%s/%064X" \
% (vol_id, random.randrange(2**128))
self.rdir.chunk_push(vol_id, container_id, content_id, chunk_id,
**args)
self.sent.add((vol_id, container_id, content_id, chunk_id))
loop -= 1
end = time.time()
self.pushed_count += loops
self.pushed_time += end - start
print("%d pushed in %.3fs, %d req/s" % (loops, end - start, loops /
(end - start)))
def harass_del(self, min_loops=0):
min_loops = min(min_loops, len(self.sent))
loops = random.randint(min_loops, len(self.sent))
print("Removing %d fake chunks" % loops)
loop = loops
start = time.time()
while loop > 0:
args = self.sent.pop()
self.rdir.chunk_delete(*args)
loop -= 1
end = time.time()
self.removed_count += loops
self.removed_time += end - start
print("%d removed in %.3fs, %d req/s" % (loops, end - start, loops /
(end - start)))
def __call__(self):
try:
while True:
self.harass_put()
self.harass_del()
except KeyboardInterrupt:
print("Cleaning...")
self.harass_del(len(self.sent))
print("Stats:")
print("Pushed %d in %.3fs, %d req/s" %
(self.pushed_count, self.pushed_time,
self.pushed_count / self.pushed_time))
print("Removed %d in %.3fs, %d req/s" %
(self.removed_count, self.removed_time,
self.removed_count / self.removed_time))
class EventWorker(Worker):
def init(self):
eventlet.monkey_patch(os=False)
self.session = requests.Session()
self.cs = ConscienceClient(self.conf)
self.rdir = RdirClient(self.conf)
self._acct_addr = None
self.acct_update = 0
self.graceful_timeout = 1
self.acct_refresh_interval = int_value(
self.conf.get('acct_refresh_interval'), 60)
self.concurrency = int_value(self.conf.get('concurrency'), 1000)
self.acct_update = true_value(self.conf.get('acct_update', True))
self.rdir_update = true_value(self.conf.get('rdir_update', True))
super(EventWorker, self).init()
def notify(self):
"""TODO"""
pass
def safe_decode_job(self, job):
try:
return json.loads(job)
except Exception as e:
self.logger.warn('ERROR decoding job "%s"', str(e.message))
return None
def run(self):
queue_url = self.conf.get('queue_url', 'tcp://127.0.0.1:11300')
self.beanstalk = Beanstalk.from_url(queue_url)
gt = eventlet.spawn(self.handle)
while self.alive:
self.notify()
try:
eventlet.sleep(1.0)
except AssertionError:
self.alive = False
break
self.notify()
try:
with Timeout(self.graceful_timeout) as t:
gt.kill(StopServe())
gt.wait()
except Timeout as te:
if te != t:
raise
gt.kill()
def handle(self):
try:
while True:
job_id, data = self.beanstalk.reserve()
try:
event = self.safe_decode_job(data)
if event:
self.process_event(event)
self.beanstalk.delete(job_id)
except Exception:
self.logger.exception("ERROR handling event %s", job_id)
except StopServe:
self.logger.info('Stopping event handler')
def process_event(self, event):
handler = self.get_handler(event)
if not handler:
self.logger.warn("ERROR no handler found for event")
# mark as success
return True
success = True
try:
handler(event)
except Exception:
success = False
finally:
return success
def get_handler(self, event):
event_type = event.get('event')
if not event_type:
return None
if event_type == EventType.CONTAINER_PUT:
return self.handle_container_put
elif event_type == EventType.CONTAINER_DESTROY:
return self.handle_container_destroy
elif event_type == EventType.CONTAINER_UPDATE:
return self.handle_container_update
elif event_type == EventType.OBJECT_PUT:
return self.handle_object_put
elif event_type == EventType.OBJECT_DELETE:
return self.handle_object_delete
elif event_type == EventType.REFERENCE_UPDATE:
return self.handle_reference_update
elif event_type == EventType.CHUNK_PUT:
return self.handle_chunk_put
elif event_type == EventType.CHUNK_DELETE:
return self.handle_chunk_delete
elif event_type == EventType.PING:
return self.handle_ping
else:
return None
@property
def acct_addr(self):
if not self._acct_addr or self.acct_refresh():
try:
acct_instance = self.cs.next_instance(ACCOUNT_SERVICE)
self._acct_addr = acct_instance.get('addr')
self.acct_update = time.time()
except Exception:
self.logger.warn('Unable to find account instance')
return self._acct_addr
def acct_refresh(self):
return (time.time() - self.acct_update) > self.acct_refresh_interval
def handle_container_put(self, event):
"""
Handle container creation.
:param event:
"""
self.logger.debug('worker handle container put')
if not self.acct_update:
return
uri = 'http://%s/v1.0/account/container/update' % self.acct_addr
mtime = event.get('when')
data = event.get('data')
name = data.get('url').get('user')
account = data.get('url').get('account')
event = {'mtime': mtime, 'name': name}
self.session.post(uri, params={'id': account}, json=event)
def handle_container_update(self, event):
"""
Handle container update.
:param event:
"""
self.logger.debug('worker handle container update')
if not self.acct_update:
return
uri = 'http://%s/v1.0/account/container/update' % self.acct_addr
mtime = event.get('when')
data = event.get('data')
name = event.get('url').get('user')
account = event.get('url').get('account')
bytes_count = data.get('bytes-count', 0)
object_count = data.get('object-count', 0)
event = {
'mtime': mtime,
'name': name,
'bytes': bytes_count,
'objects': object_count
}
self.session.post(uri, params={'id': account}, json=event)
def handle_container_destroy(self, event):
"""
Handle container destroy.
:param event:
"""
self.logger.debug('worker handle container destroy')
if not self.acct_update:
return
uri = 'http://%s/v1.0/account/container/update' % self.acct_addr
dtime = event.get('when')
data = event.get('data')
name = data.get('url').get('user')
account = data.get('url').get('account')
event = {'dtime': dtime, 'name': name}
self.session.post(uri, params={'id': account}, data=json.dumps(event))
def handle_object_delete(self, event):
"""
Handle object deletion.
Delete the chunks of the object.
:param event:
"""
self.logger.debug('worker handle object delete')
pile = GreenPile(PARALLEL_CHUNKS_DELETE)
chunks = []
for item in event.get('data'):
if item.get('type') == 'chunks':
chunks.append(item)
if not len(chunks):
self.logger.warn('No chunks found in event data')
return
def delete_chunk(chunk):
resp = None
try:
with Timeout(CHUNK_TIMEOUT):
resp = self.session.delete(chunk['id'])
except (Exception, Timeout) as e:
self.logger.warn('error while deleting chunk %s "%s"',
chunk['id'], str(e.message))
return resp
for chunk in chunks:
pile.spawn(delete_chunk, chunk)
resps = [resp for resp in pile if resp]
for resp in resps:
if resp.status_code == 204:
self.logger.debug('deleted chunk %s' % resp.url)
else:
self.logger.warn('failed to delete chunk %s' % resp.url)
def handle_object_put(self, event):
"""
Handle object creation.
TODO
:param event:
"""
self.logger.debug('worker handle object put')
def handle_reference_update(self, event):
"""
Handle reference update.
TODO
:param event
"""
self.logger.debug('worker handle reference update')
def handle_chunk_put(self, event):
"""
Handle chunk creation.
:param event
"""
if not self.rdir_update:
self.logger.debug('worker skip chunk creation')
return
self.logger.debug('worker handle chunk creation')
when = event.get('when')
data = event.get('data')
volume_id = data.get('volume_id')
del data['volume_id']
container_id = data.get('container_id')
del data['container_id']
content_id = data.get('content_id')
del data['content_id']
chunk_id = data.get('chunk_id')
del data['chunk_id']
data['mtime'] = when
self.rdir.chunk_push(volume_id, container_id, content_id, chunk_id,
**data)
def handle_chunk_delete(self, event):
"""
Handle chunk deletion.
:param event
"""
if not self.rdir_update:
self.logger.debug('worker skip chunk deletion')
return
self.logger.debug('worker handle chunk deletion')
data = event.get('data')
volume_id = data.get('volume_id')
container_id = data.get('container_id')
content_id = data.get('content_id')
chunk_id = data.get('chunk_id')
self.rdir.chunk_delete(volume_id, container_id, content_id, chunk_id)
def handle_ping(self, event):
"""
Handle ping
:param event
"""
self.logger.debug('worker handle ping')
class EventWorker(Worker):
def init(self):
eventlet.monkey_patch(os=False)
self.session = requests.Session()
self.cs = ConscienceClient(self.conf)
self.rdir = RdirClient(self.conf)
self._acct_addr = None
self.acct_update = 0
self.graceful_timeout = 1
self.acct_refresh_interval = int_value(
self.conf.get('acct_refresh_interval'), 60
)
self.concurrency = int_value(self.conf.get('concurrency'), 1000)
self.acct_update = true_value(self.conf.get('acct_update', True))
self.rdir_update = true_value(self.conf.get('rdir_update', True))
super(EventWorker, self).init()
def notify(self):
"""TODO"""
pass
def safe_decode_job(self, job):
try:
return json.loads(job)
except Exception as e:
self.logger.warn('ERROR decoding job "%s"', str(e.message))
return None
def run(self):
queue_url = self.conf.get('queue_url', 'tcp://127.0.0.1:11300')
self.beanstalk = Beanstalk.from_url(queue_url)
gt = eventlet.spawn(
self.handle)
while self.alive:
self.notify()
try:
eventlet.sleep(1.0)
except AssertionError:
self.alive = False
break
self.notify()
try:
with Timeout(self.graceful_timeout) as t:
gt.kill(StopServe())
gt.wait()
except Timeout as te:
if te != t:
raise
gt.kill()
def handle(self):
try:
while True:
job_id, data = self.beanstalk.reserve()
try:
event = self.safe_decode_job(data)
if event:
self.process_event(event)
self.beanstalk.delete(job_id)
except Exception:
self.logger.exception("ERROR handling event %s", job_id)
except StopServe:
self.logger.info('Stopping event handler')
def process_event(self, event):
handler = self.get_handler(event)
if not handler:
self.logger.warn("ERROR no handler found for event")
# mark as success
return True
success = True
try:
handler(event)
except Exception:
success = False
finally:
return success
def get_handler(self, event):
event_type = event.get('event')
if not event_type:
return None
if event_type == EventType.CONTAINER_PUT:
return self.handle_container_put
elif event_type == EventType.CONTAINER_DESTROY:
return self.handle_container_destroy
elif event_type == EventType.CONTAINER_UPDATE:
return self.handle_container_update
elif event_type == EventType.OBJECT_PUT:
return self.handle_object_put
elif event_type == EventType.OBJECT_DELETE:
return self.handle_object_delete
elif event_type == EventType.REFERENCE_UPDATE:
return self.handle_reference_update
elif event_type == EventType.CHUNK_PUT:
return self.handle_chunk_put
elif event_type == EventType.CHUNK_DELETE:
return self.handle_chunk_delete
elif event_type == EventType.PING:
return self.handle_ping
else:
return None
@property
def acct_addr(self):
if not self._acct_addr or self.acct_refresh():
try:
acct_instance = self.cs.next_instance(ACCOUNT_SERVICE)
self._acct_addr = acct_instance.get('addr')
self.acct_update = time.time()
except Exception:
self.logger.warn('Unable to find account instance')
return self._acct_addr
def acct_refresh(self):
return (time.time() - self.acct_update) > self.acct_refresh_interval
def handle_container_put(self, event):
"""
Handle container creation.
:param event:
"""
self.logger.debug('worker handle container put')
if not self.acct_update:
return
uri = 'http://%s/v1.0/account/container/update' % self.acct_addr
mtime = event.get('when')
data = event.get('data')
name = data.get('url').get('user')
account = data.get('url').get('account')
event = {'mtime': mtime, 'name': name}
self.session.post(uri, params={'id': account}, json=event)
def handle_container_update(self, event):
"""
Handle container update.
:param event:
"""
self.logger.debug('worker handle container update')
if not self.acct_update:
return
uri = 'http://%s/v1.0/account/container/update' % self.acct_addr
mtime = event.get('when')
data = event.get('data')
name = event.get('url').get('user')
account = event.get('url').get('account')
bytes_count = data.get('bytes-count', 0)
object_count = data.get('object-count', 0)
event = {
'mtime': mtime,
'name': name,
'bytes': bytes_count,
'objects': object_count
}
self.session.post(uri, params={'id': account}, json=event)
def handle_container_destroy(self, event):
"""
Handle container destroy.
:param event:
"""
self.logger.debug('worker handle container destroy')
if not self.acct_update:
return
uri = 'http://%s/v1.0/account/container/update' % self.acct_addr
dtime = event.get('when')
data = event.get('data')
name = data.get('url').get('user')
account = data.get('url').get('account')
event = {'dtime': dtime, 'name': name}
self.session.post(uri, params={'id': account}, data=json.dumps(event))
def handle_object_delete(self, event):
"""
Handle object deletion.
Delete the chunks of the object.
:param event:
"""
self.logger.debug('worker handle object delete')
pile = GreenPile(PARALLEL_CHUNKS_DELETE)
chunks = []
for item in event.get('data'):
if item.get('type') == 'chunks':
chunks.append(item)
if not len(chunks):
self.logger.warn('No chunks found in event data')
return
def delete_chunk(chunk):
resp = None
try:
with Timeout(CHUNK_TIMEOUT):
resp = self.session.delete(chunk['id'])
except (Exception, Timeout) as e:
self.logger.warn('error while deleting chunk %s "%s"',
chunk['id'], str(e.message))
return resp
for chunk in chunks:
pile.spawn(delete_chunk, chunk)
resps = [resp for resp in pile if resp]
for resp in resps:
if resp.status_code == 204:
self.logger.debug('deleted chunk %s' % resp.url)
else:
self.logger.warn('failed to delete chunk %s' % resp.url)
def handle_object_put(self, event):
"""
Handle object creation.
TODO
:param event:
"""
self.logger.debug('worker handle object put')
def handle_reference_update(self, event):
"""
Handle reference update.
TODO
:param event
"""
self.logger.debug('worker handle reference update')
def handle_chunk_put(self, event):
"""
Handle chunk creation.
:param event
"""
if not self.rdir_update:
self.logger.debug('worker skip chunk creation')
return
self.logger.debug('worker handle chunk creation')
when = event.get('when')
data = event.get('data')
volume_id = data.get('volume_id')
del data['volume_id']
container_id = data.get('container_id')
del data['container_id']
content_id = data.get('content_id')
del data['content_id']
chunk_id = data.get('chunk_id')
del data['chunk_id']
data['mtime'] = when
self.rdir.chunk_push(volume_id, container_id, content_id, chunk_id,
**data)
def handle_chunk_delete(self, event):
"""
Handle chunk deletion.
:param event
"""
if not self.rdir_update:
self.logger.debug('worker skip chunk deletion')
return
self.logger.debug('worker handle chunk deletion')
data = event.get('data')
volume_id = data.get('volume_id')
container_id = data.get('container_id')
content_id = data.get('content_id')
chunk_id = data.get('chunk_id')
self.rdir.chunk_delete(volume_id, container_id, content_id, chunk_id)
def handle_ping(self, event):
"""
Handle ping
:param event
"""
self.logger.debug('worker handle ping')
class EventWorker(object):
def __init__(self, conf, name, context, **kwargs):
self.conf = conf
self.name = name
verbose = kwargs.pop('verbose', False)
self.logger = get_logger(self.conf, verbose=verbose)
self.init_zmq(context)
self.cs = ConscienceClient(self.conf)
self.rdir = RdirClient(self.conf)
self._acct_addr = None
self.acct_update = 0
self.acct_refresh_interval = int_value(
conf.get('acct_refresh_interval'), 60)
self.acct_update = true_value(conf.get('acct_update', True))
self.rdir_update = true_value(conf.get('rdir_update', True))
self.session = requests.Session()
self.failed = False
def start(self):
self.logger.info('worker "%s" starting', self.name)
self.running = True
self.run()
def stop(self):
self.logger.info('worker "%s" stopping', self.name)
self.running = False
def init_zmq(self, context):
socket = context.socket(zmq.REP)
socket.connect('inproc://event-front')
self.socket = socket
def safe_ack(self, msg):
try:
self.socket.send_multipart(msg)
except Exception:
self.logger.warn('Unable to ack event')
def run(self):
try:
while self.running:
msg = self.socket.recv_multipart()
self.logger.debug("msg received: %s" % msg)
event = decode_msg(msg)
success = self.process_event(event)
f = "0" if success else ""
self.safe_ack([msg[0], f])
except Exception as e:
self.logger.warn('ERROR in worker "%s"', e)
self.failed = True
raise e
finally:
self.logger.info('worker "%s" stopped', self.name)
def process_event(self, event):
handler = self.get_handler(event)
if not handler:
self.logger.warn("No handler found")
# mark as success
return True
success = True
try:
handler(event)
except Exception:
success = False
finally:
return success
def get_handler(self, event):
event_type = event.get('event')
if not event_type:
return None
if event_type == EventType.CONTAINER_PUT:
return self.handle_container_put
elif event_type == EventType.CONTAINER_DESTROY:
return self.handle_container_destroy
elif event_type == EventType.CONTAINER_UPDATE:
return self.handle_container_update
elif event_type == EventType.OBJECT_PUT:
return self.handle_object_put
elif event_type == EventType.OBJECT_DELETE:
return self.handle_object_delete
elif event_type == EventType.REFERENCE_UPDATE:
return self.handle_reference_update
elif event_type == EventType.CHUNK_PUT:
return self.handle_chunk_put
elif event_type == EventType.CHUNK_DELETE:
return self.handle_chunk_delete
elif event_type == EventType.PING:
return self.handle_ping
else:
return None
@property
def acct_addr(self):
if not self._acct_addr or self.acct_refresh():
try:
acct_instance = self.cs.next_instance(ACCOUNT_SERVICE)
self._acct_addr = acct_instance.get('addr')
self.acct_update = time.time()
except Exception:
self.logger.warn('Unable to find account instance')
return self._acct_addr
def acct_refresh(self):
return (time.time() - self.acct_update) > self.acct_refresh_interval
def handle_container_put(self, event):
"""
Handle container creation.
:param event:
"""
self.logger.debug('worker "%s" handle container put', self.name)
if not self.acct_update:
return
uri = 'http://%s/v1.0/account/container/update' % self.acct_addr
mtime = event.get('when')
data = event.get('data')
name = data.get('url').get('user')
account = data.get('url').get('account')
event = {'mtime': mtime, 'name': name}
self.session.post(uri, params={'id': account}, data=json.dumps(event))
def handle_container_update(self, event):
"""
Handle container update.
:param event:
"""
self.logger.debug('worker "%s" handle container update', self.name)
if not self.acct_update:
return
uri = 'http://%s/v1.0/account/container/update' % self.acct_addr
mtime = event.get('when')
data = event.get('data')
name = event.get('url').get('user')
account = event.get('url').get('account')
bytes_count = data.get('bytes-count', 0)
object_count = data.get('object-count', 0)
event = {
'mtime': mtime,
'name': name,
'bytes': bytes_count,
'objects': object_count
}
self.session.post(uri, params={'id': account}, data=json.dumps(event))
def handle_container_destroy(self, event):
"""
Handle container destroy.
:param event:
"""
self.logger.debug('worker "%s" handle container destroy', self.name)
if not self.acct_update:
return
uri = 'http://%s/v1.0/account/container/update' % self.acct_addr
dtime = event.get('when')
data = event.get('data')
name = data.get('url').get('user')
account = data.get('url').get('account')
event = {'dtime': dtime, 'name': name}
self.session.post(uri, params={'id': account}, data=json.dumps(event))
def handle_object_delete(self, event):
"""
Handle object deletion.
Delete the chunks of the object.
:param event:
"""
self.logger.debug('worker "%s" handle object delete', self.name)
pile = GreenPile(PARALLEL_CHUNKS_DELETE)
chunks = []
for item in event.get('data'):
if item.get('type') == 'chunks':
chunks.append(item)
if not len(chunks):
self.logger.warn('No chunks found in event data')
return
def delete_chunk(chunk):
resp = None
try:
with Timeout(CHUNK_TIMEOUT):
resp = self.session.delete(chunk['id'])
except (Exception, Timeout) as e:
self.logger.exception(e)
return resp
for chunk in chunks:
pile.spawn(delete_chunk, chunk)
resps = [resp for resp in pile if resp]
for resp in resps:
if resp.status_code == 204:
self.logger.info('deleted chunk %s' % resp.url)
else:
self.logger.warn('failed to delete chunk %s' % resp.url)
def handle_object_put(self, event):
"""
Handle object creation.
TODO
:param event:
"""
self.logger.debug('worker "%s" handle object put', self.name)
def handle_reference_update(self, event):
"""
Handle reference update.
TODO
:param event
"""
self.logger.debug('worker "%s" handle reference update', self.name)
def handle_chunk_put(self, event):
"""
Handle chunk creation.
:param event
"""
if not self.rdir_update:
self.logger.debug('worker "%s" skip chunk creation', self.name)
return
self.logger.debug('worker "%s" handle chunk creation', self.name)
when = event.get('when')
data = event.get('data')
volume_id = data.get('volume_id')
del data['volume_id']
container_id = data.get('container_id')
del data['container_id']
content_id = data.get('content_id')
del data['content_id']
chunk_id = data.get('chunk_id')
del data['chunk_id']
data['mtime'] = when
self.rdir.chunk_push(volume_id, container_id, content_id, chunk_id,
**data)
def handle_chunk_delete(self, event):
"""
Handle chunk deletion.
:param event
"""
if not self.rdir_update:
self.logger.debug('worker "%s" skip chunk deletion', self.name)
return
self.logger.debug('worker "%s" handle chunk deletion', self.name)
data = event.get('data')
volume_id = data.get('volume_id')
container_id = data.get('container_id')
content_id = data.get('content_id')
chunk_id = data.get('chunk_id')
self.rdir.chunk_delete(volume_id, container_id, content_id, chunk_id)
def handle_ping(self, event):
"""
Handle ping
:param event
"""
self.logger.debug('worker "%s" handle ping', self.name)