def chunk_audit(self, path):
with open(path) as f:
try:
meta = read_chunk_metadata(f)
except exc.MissingAttribute as e:
raise exc.FaultyChunk(
'Missing extended attribute %s' % e)
size = int(meta['chunk_size'])
md5_checksum = meta['chunk_hash'].lower()
reader = ChunkReader(f, size, md5_checksum)
with closing(reader):
for buf in reader:
buf_len = len(buf)
self.bytes_running_time = ratelimit(
self.bytes_running_time,
self.max_bytes_per_second,
increment=buf_len)
self.bytes_processed += buf_len
self.total_bytes_processed += buf_len
try:
content_cid = meta['content_cid']
content_path = meta['content_path']
content_attr, data = self.container_client.content_show(
cid=content_cid, path=content_path)
# Check chunk data
chunks_nb = 0
chunk_data = None
for c in data:
if c['url'].endswith(meta['chunk_id']):
chunks_nb += 1 # FIXME: won't work with DUP
chunk_data = c
if not chunk_data:
raise exc.OrphanChunk('Not found in content')
if chunk_data['size'] != int(meta['chunk_size']):
raise exc.FaultyChunk('Invalid chunk size found')
if chunk_data['hash'] != meta['chunk_hash']:
raise exc.FaultyChunk('Invalid chunk hash found')
if chunk_data['pos'] != meta['chunk_pos']:
raise exc.FaultyChunk('Invalid chunk position found')
# Check content data
if content_attr['length'] != meta['content_size']:
raise exc.FaultyChunk('Invalid content size found')
if chunks_nb != int(meta['content_chunksnb']):
self.logger.warn('Invalid number of chunks found')
# TODO: really count chunks and enable the exception
# raise exc.FaultyChunk('Invalid number of chunks found')
except exc.NotFound:
raise exc.OrphanChunk('Chunk not found in container')
def index_pass(self):
start_time = report_time = time.time()
total_errors = 0
paths = paths_gen(self.volume)
for path in paths:
self.safe_update_index(path)
self.chunks_run_time = ratelimit(
self.chunks_run_time,
self.max_chunks_per_second
)
self.total_chunks_processed += 1
now = time.time()
if now - self.last_reported >= self.report_interval:
self.logger.info(
'%(start_time)s '
'%(passes)d '
'%(errors)d '
'%(c_rate).2f '
'%(total).2f ' % {
'start_time': time.ctime(report_time),
'passes': self.passes,
'errors': self.errors,
'c_rate': self.passes / (now - report_time),
'total': (now - start_time)
}
)
report_time = now
total_errors += self.errors
self.passes = 0
self.errors = 0
self.last_reported = now
elapsed = (time.time() - start_time) or 0.000001
self.logger.info(
'%(elapsed).02f '
'%(errors)d '
'%(chunk_rate).2f ' % {
'elapsed': elapsed,
'errors': total_errors + self.errors,
'chunk_rate': self.total_chunks_processed / elapsed
}
)
if elapsed < self.interval:
time.sleep(self.interval - elapsed)
def chunk_audit(self, path):
with open(path) as f:
try:
meta = read_chunk_metadata(f)
except exc.MissingAttribute as e:
raise exc.FaultyChunk('Missing extended attribute %s' % e)
size = int(meta['chunk_size'])
md5_checksum = meta['chunk_hash'].lower()
reader = ChunkReader(f, size, md5_checksum)
with closing(reader):
for buf in reader:
buf_len = len(buf)
self.bytes_running_time = ratelimit(
self.bytes_running_time,
self.max_bytes_per_second,
increment=buf_len)
self.bytes_processed += buf_len
self.total_bytes_processed += buf_len
try:
container_id = meta['container_id']
content_path = meta['content_path']
content_attr, data = self.container_client.content_show(
cid=container_id, path=content_path)
# Check chunk data
chunk_data = None
metachunks = set()
for c in data:
if c['url'].endswith(meta['chunk_id']):
metachunks.add(c['pos'].split('.', 2)[0])
chunk_data = c
if not chunk_data:
raise exc.OrphanChunk('Not found in content')
if chunk_data['size'] != int(meta['chunk_size']):
raise exc.FaultyChunk('Invalid chunk size found')
if chunk_data['hash'] != meta['chunk_hash']:
raise exc.FaultyChunk('Invalid chunk hash found')
if chunk_data['pos'] != meta['chunk_pos']:
raise exc.FaultyChunk('Invalid chunk position found')
except exc.NotFound:
raise exc.OrphanChunk('Chunk not found in container')
def index_pass(self):
start_time = report_time = time.time()
total_errors = 0
paths = paths_gen(self.volume)
for path in paths:
self.safe_update_index(path)
self.chunks_run_time = ratelimit(
self.chunks_run_time,
self.max_chunks_per_second
)
self.total_chunks_processed += 1
now = time.time()
if now - self.last_reported >= self.report_interval:
self.logger.info(
'%(start_time)s '
'%(passes)d '
'%(errors)d '
'%(c_rate).2f '
'%(total).2f ' % {
'start_time': time.ctime(report_time),
'passes': self.passes,
'errors': self.errors,
'c_rate': self.passes / (now - report_time),
'total': (now - start_time)
}
)
report_time = now
total_errors += self.errors
self.passes = 0
self.errors = 0
self.last_reported = now
elapsed = (time.time() - start_time) or 0.000001
self.logger.info(
'%(elapsed).02f '
'%(errors)d '
'%(chunk_rate).2f ' % {
'elapsed': elapsed,
'errors': total_errors + self.errors,
'chunk_rate': self.total_chunks_processed / elapsed
}
)
def chunk_audit(self, path):
with open(path) as f:
try:
meta = read_chunk_metadata(f)
except exc.MissingAttribute as e:
raise exc.FaultyChunk("Missing extended attribute %s" % e)
size = int(meta["chunk_size"])
md5_checksum = meta["chunk_hash"].lower()
reader = ChunkReader(f, size, md5_checksum)
with closing(reader):
for buf in reader:
buf_len = len(buf)
self.bytes_running_time = ratelimit(
self.bytes_running_time, self.max_bytes_per_second, increment=buf_len
)
self.bytes_processed += buf_len
self.total_bytes_processed += buf_len
try:
container_id = meta["container_id"]
content_path = meta["content_path"]
content_attr, data = self.container_client.content_show(cid=container_id, path=content_path)
# Check chunk data
chunk_data = None
metachunks = set()
for c in data:
if c["url"].endswith(meta["chunk_id"]):
metachunks.add(c["pos"].split(".", 2)[0])
chunk_data = c
if not chunk_data:
raise exc.OrphanChunk("Not found in content")
if chunk_data["size"] != int(meta["chunk_size"]):
raise exc.FaultyChunk("Invalid chunk size found")
if chunk_data["hash"] != meta["chunk_hash"]:
raise exc.FaultyChunk("Invalid chunk hash found")
if chunk_data["pos"] != meta["chunk_pos"]:
raise exc.FaultyChunk("Invalid chunk position found")
except exc.NotFound:
raise exc.OrphanChunk("Chunk not found in container")
def run(self):
start_time = report_time = time.time()
total_errors = 0
for (container_id, content_id) in self._list_contents():
self.safe_change_policy(container_id, content_id)
self.contents_run_time = ratelimit(
self.contents_run_time,
self.max_contents_per_second
)
self.total_contents_processed += 1
now = time.time()
if now - self.last_reported >= self.report_interval:
self.logger.info(
'%(start_time)s '
'%(passes)d '
'%(errors)d '
'%(c_rate).2f '
'%(total).2f ' % {
'start_time': time.ctime(report_time),
'passes': self.passes,
'errors': self.errors,
'c_rate': self.passes / (now - report_time),
'total': (now - start_time)
}
)
report_time = now
total_errors += self.errors
self.passes = 0
self.errors = 0
self.last_reported = now
elapsed = (time.time() - start_time) or 0.000001
self.logger.info(
'%(elapsed).02f '
'%(errors)d '
'%(content_rate).2f ' % {
'elapsed': elapsed,
'errors': total_errors + self.errors,
'content_rate': self.total_contents_processed / elapsed
}
)
def chunk_audit(self, path):
with open(path) as f:
try:
meta = read_chunk_metadata(f)
except exc.MissingAttribute as e:
raise exc.FaultyChunk(
'Missing extended attribute %s' % e)
size = int(meta['chunk_size'])
md5_checksum = meta['chunk_hash'].lower()
reader = ChunkReader(f, size, md5_checksum)
with closing(reader):
for buf in reader:
buf_len = len(buf)
self.bytes_running_time = ratelimit(
self.bytes_running_time,
self.max_bytes_per_second,
increment=buf_len)
self.bytes_processed += buf_len
self.total_bytes_processed += buf_len
try:
content_cid = meta['content_cid']
content_path = meta['content_path']
_, data = self.container_client.content_show(
cid=content_cid, path=content_path)
chunk_data = None
for c in data:
if c['url'].endswith(meta['chunk_id']):
chunk_data = c
if not chunk_data:
raise exc.OrphanChunk('Not found in content')
if chunk_data['size'] != int(meta['chunk_size']):
raise exc.FaultyChunk('Invalid chunk size found')
if chunk_data['hash'] != meta['chunk_hash']:
raise exc.FaultyChunk('Invalid chunk hash found')
if chunk_data['pos'] != meta['chunk_pos']:
raise exc.FaultyChunk('Invalid chunk position found')
except exc.NotFound:
raise exc.OrphanChunk('Chunk not found in container')
def run(self):
start_time = report_time = time.time()
total_errors = 0
for (container_id, content_id) in self._list_contents():
self.safe_change_policy(container_id, content_id)
self.contents_run_time = ratelimit(self.contents_run_time,
self.max_contents_per_second)
self.total_contents_processed += 1
now = time.time()
if now - self.last_reported >= self.report_interval:
self.logger.info(
'%(start_time)s '
'%(passes)d '
'%(errors)d '
'%(c_rate).2f '
'%(total).2f ' % {
'start_time': time.ctime(report_time),
'passes': self.passes,
'errors': self.errors,
'c_rate': self.passes / (now - report_time),
'total': (now - start_time)
})
report_time = now
total_errors += self.errors
self.passes = 0
self.errors = 0
self.last_reported = now
elapsed = (time.time() - start_time) or 0.000001
self.logger.info(
'%(elapsed).02f '
'%(errors)d '
'%(content_rate).2f ' % {
'elapsed': elapsed,
'errors': total_errors + self.errors,
'content_rate': self.total_contents_processed / elapsed
})
def rebuilder_pass(self):
start_time = report_time = time.time()
total_errors = 0
rebuilder_time = 0
chunks = self.rdir_client.chunk_fetch(self.volume,
limit=self.rdir_fetch_limit,
rebuild=True)
for container_id, content_id, chunk_id, data in chunks:
loop_time = time.time()
if self.dry_run:
self.dryrun_chunk_rebuild(container_id, content_id, chunk_id)
else:
self.safe_chunk_rebuild(container_id, content_id, chunk_id)
self.chunks_run_time = ratelimit(
self.chunks_run_time,
self.max_chunks_per_second
)
self.total_chunks_processed += 1
now = time.time()
if now - self.last_reported >= self.report_interval:
self.logger.info(
'%(start_time)s '
'%(passes)d '
'%(errors)d '
'%(c_rate).2f '
'%(b_rate).2f '
'%(total).2f '
'%(rebuilder_time).2f'
'%(rebuilder_rate).2f' % {
'start_time': time.ctime(report_time),
'passes': self.passes,
'errors': self.errors,
'c_rate': self.passes / (now - report_time),
'b_rate': self.bytes_processed / (now - report_time),
'total': (now - start_time),
'rebuilder_time': rebuilder_time,
'rebuilder_rate': rebuilder_time / (now - start_time)
}
)
report_time = now
total_errors += self.errors
self.passes = 0
self.bytes_processed = 0
self.last_reported = now
rebuilder_time += (now - loop_time)
elapsed = (time.time() - start_time) or 0.000001
self.logger.info(
'%(elapsed).02f '
'%(errors)d '
'%(chunk_rate).2f '
'%(bytes_rate).2f '
'%(rebuilder_time).2f '
'%(rebuilder_rate).2f' % {
'elapsed': elapsed,
'errors': total_errors + self.errors,
'chunk_rate': self.total_chunks_processed / elapsed,
'bytes_rate': self.total_bytes_processed / elapsed,
'rebuilder_time': rebuilder_time,
'rebuilder_rate': rebuilder_time / elapsed
}
)
def audit_pass(self):
self.namespace, self.address = check_volume(self.volume)
start_time = report_time = time.time()
total_errors = 0
total_corrupted = 0
total_orphans = 0
total_faulty = 0
audit_time = 0
paths = paths_gen(self.volume)
for path in paths:
loop_time = time.time()
self.safe_chunk_audit(path)
self.chunks_run_time = ratelimit(
self.chunks_run_time,
self.max_chunks_per_second
)
self.total_chunks_processed += 1
now = time.time()
if now - self.last_reported >= self.report_interval:
self.logger.info(
'%(start_time)s '
'%(passes)d '
'%(corrupted)d '
'%(faulty)d '
'%(orphans)d '
'%(errors)d '
'%(c_rate).2f '
'%(b_rate).2f '
'%(total).2f '
'%(audit_time).2f'
'%(audit_rate).2f' % {
'start_time': time.ctime(report_time),
'passes': self.passes,
'corrupted': self.corrupted_chunks,
'faulty': self.faulty_chunks,
'orphans': self.orphan_chunks,
'errors': self.errors,
'c_rate': self.passes / (now - report_time),
'b_rate': self.bytes_processed / (now - report_time),
'total': (now - start_time),
'audit_time': audit_time,
'audit_rate': audit_time / (now - start_time)
}
)
report_time = now
total_corrupted += self.corrupted_chunks
total_orphans += self.orphan_chunks
total_faulty += self.faulty_chunks
total_errors += self.errors
self.passes = 0
self.corrupted_chunks = 0
self.orphan_chunks = 0
self.faulty_chunks = 0
self.errors = 0
self.bytes_processed = 0
self.last_reported = now
audit_time += (now - loop_time)
elapsed = (time.time() - start_time) or 0.000001
self.logger.info(
'%(elapsed).02f '
'%(corrupted)d '
'%(faulty)d '
'%(orphans)d '
'%(errors)d '
'%(chunk_rate).2f '
'%(bytes_rate).2f '
'%(audit_time).2f '
'%(audit_rate).2f' % {
'elapsed': elapsed,
'corrupted': total_corrupted + self.corrupted_chunks,
'faulty': total_faulty + self.faulty_chunks,
'orphans': total_orphans + self.orphan_chunks,
'errors': total_errors + self.errors,
'chunk_rate': self.total_chunks_processed / elapsed,
'bytes_rate': self.total_bytes_processed / elapsed,
'audit_time': audit_time,
'audit_rate': audit_time / elapsed
}
)
def audit_pass(self):
self.namespace, self.address = check_volume(self.volume)
start_time = report_time = time.time()
total_errors = 0
total_corrupted = 0
total_orphans = 0
total_faulty = 0
audit_time = 0
paths = paths_gen(self.volume)
for path in paths:
loop_time = time.time()
self.safe_chunk_audit(path)
self.chunks_run_time = ratelimit(self.chunks_run_time, self.max_chunks_per_second)
self.total_chunks_processed += 1
now = time.time()
if now - self.last_reported >= self.report_interval:
self.logger.info(
"%(start_time)s "
"%(passes)d "
"%(corrupted)d "
"%(faulty)d "
"%(orphans)d "
"%(errors)d "
"%(c_rate).2f "
"%(b_rate).2f "
"%(total).2f "
"%(audit_time).2f"
"%(audit_rate).2f"
% {
"start_time": time.ctime(report_time),
"passes": self.passes,
"corrupted": self.corrupted_chunks,
"faulty": self.faulty_chunks,
"orphans": self.orphan_chunks,
"errors": self.errors,
"c_rate": self.passes / (now - report_time),
"b_rate": self.bytes_processed / (now - report_time),
"total": (now - start_time),
"audit_time": audit_time,
"audit_rate": audit_time / (now - start_time),
}
)
report_time = now
total_corrupted += self.corrupted_chunks
total_orphans += self.orphan_chunks
total_faulty += self.faulty_chunks
total_errors += self.errors
self.passes = 0
self.corrupted_chunks = 0
self.orphan_chunks = 0
self.faulty_chunks = 0
self.errors = 0
self.bytes_processed = 0
self.last_reported = now
audit_time += now - loop_time
elapsed = (time.time() - start_time) or 0.000001
self.logger.info(
"%(elapsed).02f "
"%(corrupted)d "
"%(faulty)d "
"%(orphans)d "
"%(errors)d "
"%(chunk_rate).2f "
"%(bytes_rate).2f "
"%(audit_time).2f "
"%(audit_rate).2f"
% {
"elapsed": elapsed,
"corrupted": total_corrupted + self.corrupted_chunks,
"faulty": total_faulty + self.faulty_chunks,
"orphans": total_orphans + self.orphan_chunks,
"errors": total_errors + self.errors,
"chunk_rate": self.total_chunks_processed / elapsed,
"bytes_rate": self.total_bytes_processed / elapsed,
"audit_time": audit_time,
"audit_rate": audit_time / elapsed,
}
)
def mover_pass(self):
self.namespace, self.address = check_volume(self.volume)
start_time = report_time = time.time()
total_errors = 0
mover_time = 0
paths = paths_gen(self.volume)
for path in paths:
loop_time = time.time()
now = time.time()
if now - self.last_usage_check >= self.usage_check_interval:
used, total = statfs(self.volume)
usage = (float(used) / total) * 100
if usage <= self.usage_target:
self.logger.info(
'current usage %.2f%%: target reached (%.2f%%)', usage,
self.usage_target)
self.last_usage_check = now
break
self.safe_chunk_move(path)
self.chunks_run_time = ratelimit(self.chunks_run_time,
self.max_chunks_per_second)
self.total_chunks_processed += 1
now = time.time()
if now - self.last_reported >= self.report_interval:
self.logger.info(
'%(start_time)s '
'%(passes)d '
'%(errors)d '
'%(c_rate).2f '
'%(b_rate).2f '
'%(total).2f '
'%(mover_time).2f'
'%(mover_rate).2f' % {
'start_time': time.ctime(report_time),
'passes': self.passes,
'errors': self.errors,
'c_rate': self.passes / (now - report_time),
'b_rate': self.bytes_processed / (now - report_time),
'total': (now - start_time),
'mover_time': mover_time,
'mover_rate': mover_time / (now - start_time)
})
report_time = now
total_errors += self.errors
self.passes = 0
self.bytes_processed = 0
self.last_reported = now
mover_time += (now - loop_time)
elapsed = (time.time() - start_time) or 0.000001
self.logger.info(
'%(elapsed).02f '
'%(errors)d '
'%(chunk_rate).2f '
'%(bytes_rate).2f '
'%(mover_time).2f '
'%(mover_rate).2f' % {
'elapsed': elapsed,
'errors': total_errors + self.errors,
'chunk_rate': self.total_chunks_processed / elapsed,
'bytes_rate': self.total_bytes_processed / elapsed,
'mover_time': mover_time,
'mover_rate': mover_time / elapsed
})
def mover_pass(self):
self.namespace, self.address = check_volume(self.volume)
start_time = report_time = time.time()
total_errors = 0
mover_time = 0
paths = paths_gen(self.volume)
for path in paths:
loop_time = time.time()
now = time.time()
if now - self.last_usage_check >= self.usage_check_interval:
used, total = statfs(self.volume)
usage = (float(used) / total) * 100
if usage <= self.usage_target:
self.logger.info(
'current usage %.2f%%: target reached (%.2f%%)', usage,
self.usage_target)
self.last_usage_check = now
break
self.safe_chunk_move(path)
self.chunks_run_time = ratelimit(
self.chunks_run_time,
self.max_chunks_per_second
)
self.total_chunks_processed += 1
now = time.time()
if now - self.last_reported >= self.report_interval:
self.logger.info(
'%(start_time)s '
'%(passes)d '
'%(errors)d '
'%(c_rate).2f '
'%(b_rate).2f '
'%(total).2f '
'%(mover_time).2f'
'%(mover_rate).2f' % {
'start_time': time.ctime(report_time),
'passes': self.passes,
'errors': self.errors,
'c_rate': self.passes / (now - report_time),
'b_rate': self.bytes_processed / (now - report_time),
'total': (now - start_time),
'mover_time': mover_time,
'mover_rate': mover_time / (now - start_time)
}
)
report_time = now
total_errors += self.errors
self.passes = 0
self.bytes_processed = 0
self.last_reported = now
mover_time += (now - loop_time)
elapsed = (time.time() - start_time) or 0.000001
self.logger.info(
'%(elapsed).02f '
'%(errors)d '
'%(chunk_rate).2f '
'%(bytes_rate).2f '
'%(mover_time).2f '
'%(mover_rate).2f' % {
'elapsed': elapsed,
'errors': total_errors + self.errors,
'chunk_rate': self.total_chunks_processed / elapsed,
'bytes_rate': self.total_bytes_processed / elapsed,
'mover_time': mover_time,
'mover_rate': mover_time / elapsed
}
)
def audit_pass(self):
self.namespace, self.address = check_volume(self.volume)
start_time = report_time = time.time()
total_errors = 0
total_corrupted = 0
total_orphans = 0
total_faulty = 0
audit_time = 0
paths = paths_gen(self.volume)
for path in paths:
loop_time = time.time()
self.safe_chunk_audit(path)
self.chunks_run_time = ratelimit(self.chunks_run_time,
self.max_chunks_per_second)
self.total_chunks_processed += 1
now = time.time()
if now - self.last_reported >= self.report_interval:
self.logger.info(
'%(start_time)s '
'%(passes)d '
'%(corrupted)d '
'%(faulty)d '
'%(orphans)d '
'%(errors)d '
'%(c_rate).2f '
'%(b_rate).2f '
'%(total).2f '
'%(audit_time).2f'
'%(audit_rate).2f' % {
'start_time': time.ctime(report_time),
'passes': self.passes,
'corrupted': self.corrupted_chunks,
'faulty': self.faulty_chunks,
'orphans': self.orphan_chunks,
'errors': self.errors,
'c_rate': self.passes / (now - report_time),
'b_rate': self.bytes_processed / (now - report_time),
'total': (now - start_time),
'audit_time': audit_time,
'audit_rate': audit_time / (now - start_time)
})
report_time = now
total_corrupted += self.corrupted_chunks
total_orphans += self.orphan_chunks
total_faulty += self.faulty_chunks
total_errors += self.errors
self.passes = 0
self.corrupted_chunks = 0
self.orphan_chunks = 0
self.faulty_chunks = 0
self.errors = 0
self.bytes_processed = 0
self.last_reported = now
audit_time += (now - loop_time)
elapsed = (time.time() - start_time) or 0.000001
self.logger.info(
'%(elapsed).02f '
'%(corrupted)d '
'%(faulty)d '
'%(orphans)d '
'%(errors)d '
'%(chunk_rate).2f '
'%(bytes_rate).2f '
'%(audit_time).2f '
'%(audit_rate).2f' % {
'elapsed': elapsed,
'corrupted': total_corrupted + self.corrupted_chunks,
'faulty': total_faulty + self.faulty_chunks,
'orphans': total_orphans + self.orphan_chunks,
'errors': total_errors + self.errors,
'chunk_rate': self.total_chunks_processed / elapsed,
'bytes_rate': self.total_bytes_processed / elapsed,
'audit_time': audit_time,
'audit_rate': audit_time / elapsed
})
def mover_pass(self):
self.namespace, self.address = check_volume(self.volume)
start_time = report_time = time.time()
total_errors = 0
mover_time = 0
paths = paths_gen(self.volume)
for path in paths:
loop_time = time.time()
now = time.time()
if now - self.last_usage_check >= self.usage_check_interval:
used, total = statfs(self.volume)
usage = (float(used) / total) * 100
if usage <= self.usage_target:
self.logger.info("current usage %.2f%%: target reached (%.2f%%)", usage, self.usage_target)
self.last_usage_check = now
break
self.safe_chunk_move(path)
self.chunks_run_time = ratelimit(self.chunks_run_time, self.max_chunks_per_second)
self.total_chunks_processed += 1
now = time.time()
if now - self.last_reported >= self.report_interval:
self.logger.info(
"%(start_time)s "
"%(passes)d "
"%(errors)d "
"%(c_rate).2f "
"%(b_rate).2f "
"%(total).2f "
"%(mover_time).2f"
"%(mover_rate).2f"
% {
"start_time": time.ctime(report_time),
"passes": self.passes,
"errors": self.errors,
"c_rate": self.passes / (now - report_time),
"b_rate": self.bytes_processed / (now - report_time),
"total": (now - start_time),
"mover_time": mover_time,
"mover_rate": mover_time / (now - start_time),
}
)
report_time = now
total_errors += self.errors
self.passes = 0
self.bytes_processed = 0
self.last_reported = now
mover_time += now - loop_time
elapsed = (time.time() - start_time) or 0.000001
self.logger.info(
"%(elapsed).02f "
"%(errors)d "
"%(chunk_rate).2f "
"%(bytes_rate).2f "
"%(mover_time).2f "
"%(mover_rate).2f"
% {
"elapsed": elapsed,
"errors": total_errors + self.errors,
"chunk_rate": self.total_chunks_processed / elapsed,
"bytes_rate": self.total_bytes_processed / elapsed,
"mover_time": mover_time,
"mover_rate": mover_time / elapsed,
}
)
def run(self, *args, **kwargs):
try:
self.logger.info('event agent: starting')
pool = GreenPool(len(self.workers))
for worker in self.workers:
pool.spawn(worker.start)
def front(server, backend):
while True:
msg = server.recv_multipart()
if validate_msg(msg):
try:
event_id = msg[2]
data = msg[3]
self.queue.put(event_id, data)
event = ['', msg[2], msg[3]]
backend.send_multipart(event)
except Exception:
pass
finally:
ack = msg[0:3]
server.send_multipart(ack)
def back(backend):
while True:
msg = backend.recv_multipart()
event_id = msg[1]
success = msg[2]
if not success:
self.queue.failed(event_id)
self.logger.warn('event %s moved to failed',
binascii.hexlify(event_id))
else:
self.queue.delete(event_id)
self.logger.debug('event %s removed from queue',
binascii.hexlify(event_id))
boss_pool = GreenPool(2)
boss_pool.spawn_n(front, self.server, self.backend)
boss_pool.spawn_n(back, self.backend)
while True:
results = self.queue.load(self.batch_size)
for event in results:
event_id, data = event
msg = ['', event_id, str(data)]
self.backend.send_multipart(msg)
self.retries_run_time = ratelimit(
self.retries_run_time, self.max_retries_per_second)
for w in self.workers:
if w.failed:
self.workers.remove(w)
self.logger.warn('restart worker "%s"', w.name)
new_w = EventWorker(self.conf, w.name, self.context)
self.workers.append(new_w)
pool.spawn(new_w.start)
sleep(SLEEP_TIME)
except Exception as e:
self.logger.error('ERROR in main loop %s', e)
raise
finally:
self.logger.warn('event agent: stopping')
self.stop_workers()
self.context.destroy(linger=True)
self.context = None
def rebuilder_pass(self):
start_time = report_time = time.time()
total_errors = 0
rebuilder_time = 0
chunks = self.rdir_client.chunk_fetch(self.volume,
limit=self.rdir_fetch_limit,
rebuild=True)
for container_id, content_id, chunk_id, data in chunks:
loop_time = time.time()
if self.dry_run:
self.dryrun_chunk_rebuild(container_id, content_id, chunk_id)
else:
self.safe_chunk_rebuild(container_id, content_id, chunk_id)
self.chunks_run_time = ratelimit(self.chunks_run_time,
self.max_chunks_per_second)
self.total_chunks_processed += 1
now = time.time()
if now - self.last_reported >= self.report_interval:
self.logger.info(
'%(start_time)s '
'%(passes)d '
'%(errors)d '
'%(c_rate).2f '
'%(b_rate).2f '
'%(total).2f '
'%(rebuilder_time).2f'
'%(rebuilder_rate).2f' % {
'start_time': time.ctime(report_time),
'passes': self.passes,
'errors': self.errors,
'c_rate': self.passes / (now - report_time),
'b_rate': self.bytes_processed / (now - report_time),
'total': (now - start_time),
'rebuilder_time': rebuilder_time,
'rebuilder_rate': rebuilder_time / (now - start_time)
})
report_time = now
total_errors += self.errors
self.passes = 0
self.bytes_processed = 0
self.last_reported = now
rebuilder_time += (now - loop_time)
elapsed = (time.time() - start_time) or 0.000001
self.logger.info(
'%(elapsed).02f '
'%(errors)d '
'%(chunk_rate).2f '
'%(bytes_rate).2f '
'%(rebuilder_time).2f '
'%(rebuilder_rate).2f' % {
'elapsed': elapsed,
'errors': total_errors + self.errors,
'chunk_rate': self.total_chunks_processed / elapsed,
'bytes_rate': self.total_bytes_processed / elapsed,
'rebuilder_time': rebuilder_time,
'rebuilder_rate': rebuilder_time / elapsed
})
