def merge_files(meta, metrics, data_dir, output_file):
contents = [get_whisper_file_content(data_dir, m) for m in metrics]
mkdir_p(os.path.dirname(output_file))
needed_metrics = meta['metrics_max_num'] - len(metrics)
now = int(time.time())
with open(output_file, 'w') as f:
archives = meta['archives']
archive_info = [(archive['sec_per_point'], archive['count'])
for archive in archives]
agg_name = get_agg_name(meta['agg_type'])
inter_tag_list = metrics + [''] * (needed_metrics + 1)
# header
packed_kenshin_header = Storage.pack_header(inter_tag_list,
archive_info, meta['xff'],
agg_name)[0]
f.write(packed_kenshin_header)
# archives
for archive in archives:
whisper_points = [
read_whisper_points(content, archive, now)
for content in contents
]
archive_points = merge_points(whisper_points, needed_metrics)
archive_points = fill_gap(archive_points, archive,
meta['metrics_max_num'])
packed_str = packed_kenshin_points(archive_points)
f.write(packed_str)
def create(self,
metric_name,
tag_list,
archive_list,
x_files_factor=None,
agg_name=None):
Storage.validate_archive_list(archive_list, x_files_factor)
path = self.gen_path(self.data_dir, metric_name)
if os.path.exists(path):
raise IOError('file %s already exits.' % path)
else:
mkdir_p(os.path.dirname(path))
# inter_tag_list[RESERVED_INDEX] is reserved space
# to avoid move data points.
empty_tag_cnt = sum(1 for t in tag_list if not t)
inter_tag_list = tag_list + ['N' * DEFAULT_TAG_LENGTH * empty_tag_cnt]
with open(path, 'wb') as f:
packed_header, end_offset = self.pack_header(
inter_tag_list, archive_list, x_files_factor, agg_name)
f.write(packed_header)
# init data
remaining = end_offset - f.tell()
zeroes = '\x00' * CHUNK_SIZE
while remaining > CHUNK_SIZE:
f.write(zeroes)
remaining -= CHUNK_SIZE
f.write(zeroes[:remaining])
def create_link(metric, link_dir, file_path):
link_path = metric.replace('.', os.path.sep)
link_path = os.path.join(link_dir, link_path + '.hs')
dirname = os.path.dirname(link_path)
mkdir_p(dirname)
if os.path.exists(link_path):
os.remove(link_path)
os.symlink(file_path, link_path)
def _createLinkHelper(link_path, file_path):
"""
Create symlink link_path -> file_path.
"""
dir_ = dirname(link_path)
mkdir_p(dir_)
if os.path.lexists(link_path):
os.rename(link_path, link_path + '.bak')
os.symlink(file_path, link_path)
def _createLinkHelper(link_path, file_path):
"""
Create symlink link_path -> file_path.
"""
dir_ = dirname(link_path)
mkdir_p(dir_)
if os.path.lexists(link_path):
os.rename(link_path, link_path +'.bak')
os.symlink(file_path, link_path)
def gen_links(metrics, data_path, link_dir, instance):
for m in metrics:
link_path = m.replace('.', os.path.sep)
link_path = os.path.join(link_dir, instance, link_path + '.hs')
dirname = os.path.dirname(link_path)
mkdir_p(dirname)
if os.path.exists(link_path):
os.remove(link_path)
os.symlink(data_path, link_path)
def setUp(self):
if os.path.exists(self.data_dir):
shutil.rmtree(self.data_dir)
mkdir_p(self.data_dir)
self.storage = Storage(data_dir=self.data_dir)
self.basic_setup = self._basic_setup()
self.storage.create(*self.basic_setup)
metric_name = self.basic_setup[0]
self.path = self.storage.gen_path(self.data_dir, metric_name)
def setUp(self):
if os.path.exists(self.data_dir):
shutil.rmtree(self.data_dir)
mkdir_p(self.data_dir)
self.storage = Storage(data_dir=self.data_dir)
self.basic_setup = self._basic_setup()
self.storage.create(*self.basic_setup)
metric_name = self.basic_setup[0]
self.path = self.storage.gen_path(self.data_dir, metric_name)
tag_list = self.basic_setup[1]
self.null_point = (None,) * len(tag_list)