def on_benchmark_stop(self):
logger.info("Analyzing merge times.")
# first decompress all logs. They have unique names so it's safe to do that. It's easier to first decompress everything
for log_file in os.listdir(self.node_log_dir):
log_path = "%s/%s" % (self.node_log_dir, log_file)
if io.is_archive(log_path):
logger.info("Decompressing [%s] to analyze merge times..." %
log_path)
io.decompress(log_path, self.node_log_dir)
# we need to add up times from all files
merge_times = {}
for log_file in os.listdir(self.node_log_dir):
log_path = "%s/%s" % (self.node_log_dir, log_file)
if not io.is_archive(log_file):
logger.debug("Analyzing merge times in [%s]" % log_path)
with open(log_path, mode="rt", encoding="utf-8") as f:
self._extract_merge_times(f, merge_times)
else:
logger.debug("Skipping archived logs in [%s]." % log_path)
if merge_times:
self._store_merge_times(merge_times)
logger.info(
"Finished analyzing merge times. Extracted [%s] different merge time components."
% len(merge_times))
def test_archive(self):
self.assertTrue(io.is_archive("/tmp/some-archive.tar.gz"))
self.assertTrue(io.is_archive("/tmp/some-archive.tgz"))
# Rally does not recognize .7z
self.assertFalse(io.is_archive("/tmp/some-archive.7z"))
self.assertFalse(io.is_archive("/tmp/some.log"))
self.assertFalse(io.is_archive("some.log"))
def test_archive(self):
assert io.is_archive("/tmp/some-archive.tar.gz")
assert io.is_archive("/tmp/some-archive.tgz")
# Rally does not recognize .7z
assert not io.is_archive("/tmp/some-archive.7z")
assert not io.is_archive("/tmp/some.log")
assert not io.is_archive("some.log")
def _create_type(self, type_spec, mapping_dir):
docs = self._r(type_spec, "documents", mandatory=False)
if docs:
if io.is_archive(docs):
document_archive = docs
document_file = io.splitext(docs)[0]
else:
document_archive = None
document_file = docs
number_of_documents = self._r(type_spec, "document-count")
compressed_bytes = self._r(type_spec, "compressed-bytes", mandatory=False)
uncompressed_bytes = self._r(type_spec, "uncompressed-bytes", mandatory=False)
else:
document_archive = None
document_file = None
number_of_documents = 0
compressed_bytes = 0
uncompressed_bytes = 0
mapping_file = os.path.join(mapping_dir, self._r(type_spec, "mapping"))
with self.source(mapping_file, "rt") as f:
mapping = json.load(f)
return track.Type(name=self._r(type_spec, "name"),
mapping=mapping,
document_file=document_file,
document_archive=document_archive,
includes_action_and_meta_data=self._r(type_spec, "includes-action-and-meta-data", mandatory=False,
default_value=False),
number_of_documents=number_of_documents,
compressed_size_in_bytes=compressed_bytes,
uncompressed_size_in_bytes=uncompressed_bytes)