def create_quota_trend_chart(local_mode, fs_long_name, chart_dir, start_date,
end_date, quota_trend_chart, quota_history_table):
if local_mode:
item_list = ih.create_dummy_group_date_values(50, 200)
else:
groups = get_user_groups()
item_list = \
quota_history_table.get_time_series_group_quota_usage(start_date,
end_date,
groups)
group_item_dict = ih.create_group_date_value_item_dict(item_list)
data_frame = create_data_frame_weekly(group_item_dict)
title = "Group Quota Trend on %s" % fs_long_name
chart_path = chart_dir + os.path.sep + quota_trend_chart
chart = TrendChart(title, data_frame, chart_path, 'Time (Weeks)',
'Quota Used (%)')
chart.create()
return chart_path
def create_usage_trend_chart(local_mode, fs_long_name, chart_dir, start_date,
end_date, threshold, usage_trend_chart,
quota_history_table):
if local_mode:
item_list = ih.create_dummy_group_date_values(8, 1000)
else:
groups = get_user_groups()
filtered_groups = \
quota_history_table.filter_groups_at_threshold(start_date,
end_date,
threshold,
groups)
item_list = \
quota_history_table.get_time_series_group_sizes(
start_date,
end_date,
filtered_groups)
group_item_dict = ih.create_group_date_value_item_dict(item_list)
data_frame = create_data_frame_weekly(group_item_dict)
title = "Top Groups Usage Trend on %s" % fs_long_name
chart_path = chart_dir + os.path.sep + usage_trend_chart
chart = TrendChart(title, data_frame, chart_path, 'Time (Weeks)',
'Disk Space Used (TiB)')
chart.create()
return chart_path
def create_weekly_reports(local_mode,
chart_dir,
file_system,
fs_long_name,
quota_pct_bar_chart,
usage_quota_bar_chart,
usage_pie_chart,
num_top_groups):
reports_path_list = list()
group_info_list = None
storage_total_size = 0
if local_mode:
# TODO: create dummy list with variable parameter
group_info_list = ih.create_dummy_group_info_list()
storage_total_size = 18458963071860736
else:
group_names_list = get_user_groups()
group_info_list = \
gf.filter_group_info_items(
ldh.create_group_info_list(group_names_list, file_system))
storage_total_size = ldh.lustre_total_size(file_system)
# QUOTA-PCT-BAR-CHART
title = "Group Quota Usage on %s" % fs_long_name
chart_path = chart_dir + os.path.sep + quota_pct_bar_chart
chart = QuotaPctBarChart(title, group_info_list, chart_path)
chart.create()
logging.debug("Created chart: %s" % chart_path)
reports_path_list.append(chart_path)
# USAGE-QUOTA-BAR-CHART
title = "Quota and Disk Space Usage on %s" % fs_long_name
chart_path = chart_dir + os.path.sep + usage_quota_bar_chart
chart = UsageQuotaBarChart(title, group_info_list, chart_path)
chart.create()
logging.debug("Created chart: %s" % chart_path)
reports_path_list.append(chart_path)
# USAGE-PIE-CHART
title = "Storage Usage on %s" % fs_long_name
chart_path = chart_dir + os.path.sep + usage_pie_chart
chart = UsagePieChart(title,
group_info_list,
chart_path,
storage_total_size,
num_top_groups)
chart.create()
logging.debug("Created chart: %s" % chart_path)
reports_path_list.append(chart_path)
return reports_path_list
def create_report(local_mode, chart_dir, fs1_name, fs2_name, config):
reports_path_list = list()
group_info_list = list()
if local_mode:
group_info_list = ih.create_dummy_group_files_migration_info_list()
else:
group_names = get_user_groups()
files_threshold = int(config.get(
'group_files_migration_bar_chart', 'files_threshold'))
fs1 = config.get('storage', 'file_system_1')
fs2 = config.get('storage', 'file_system_2')
g1_info_list = create_group_info_list(group_names, fs1)
g2_info_list = create_group_info_list(group_names, fs2)
group1_info_items = gf.filter_group_info_items(g1_info_list)
group2_info_items = gf.filter_group_info_items(g2_info_list)
for gid in group_names:
fs1_files = Decimal(0)
fs2_files = Decimal(0)
for group1_info_item in group1_info_items:
if gid in group1_info_item.name:
fs1_files = group1_info_item.files
break
for group2_info_item in group2_info_items:
if gid in group2_info_item.name:
fs2_files = group2_info_item.files
break
if fs1_files > files_threshold or fs2_files > files_threshold:
logging.debug("Append GroupFilesMigrationInfoItem(%s, %s, %s)" %
(gid, fs1_files, fs2_files))
group_info_list.append(ih.GroupFilesMigrationInfoItem(
gid, fs1_files, fs2_files))
# GROUP-FILES-MIGRATION-BAR-CHART
title = "Group Files Migration Lustre Nyx and Hebe"
chart_path = chart_dir + os.path.sep + config.get( \
'group_files_migration_bar_chart', 'filename')
chart = GroupFilesMigrationBarChart(title, group_info_list, chart_path,
fs1_name, fs2_name)
chart.create()
logging.debug("Created chart: %s" % chart_path)
reports_path_list.append(chart_path)
return reports_path_list
def main():
# Default run-mode: collect
run_mode = 'collect'
parser = argparse.ArgumentParser(description='')
parser.add_argument('-f',
'--config-file',
dest='config_file',
type=str,
required=True,
help='Path of the config file.')
parser.add_argument(
'-m',
'--run-mode',
dest='run_mode',
type=str,
default=run_mode,
required=False,
help="Specifies the run mode: 'print' or 'collect' - Default: %s" %
run_mode)
parser.add_argument('-D',
'--enable-debug',
dest='enable_debug',
required=False,
action='store_true',
help='Enables logging of debug messages.')
parser.add_argument('--create-table',
dest='create_table',
required=False,
action='store_true',
help='Creates the group quota history table.')
args = parser.parse_args()
if not os.path.isfile(args.config_file):
raise IOError("The config file does not exist or is not a file: %s" %
args.config_file)
logging_level = logging.ERROR
if args.enable_debug:
logging_level = logging.DEBUG
logging.basicConfig(level=logging_level,
format='%(asctime)s - %(levelname)s: %(message)s')
if not (args.run_mode == 'print' or args.run_mode == 'collect'):
raise RuntimeError("Invalid run mode: %s" % args.run_mode)
else:
run_mode = args.run_mode
try:
logging.info('START')
date_today = time.strftime('%Y-%m-%d')
config = configparser.ConfigParser()
config.read(args.config_file)
if args.create_table:
create_group_quota_history_table(config)
logging.info('END')
sys.exit(0)
fs = config.get('lustre', 'file_system')
group_info_list = ldh.create_group_info_list(get_user_groups(), fs)
if run_mode == 'print':
for group_info in group_info_list:
print("Group: %s - Used: %s - Quota: %s - Files: %s" \
% (group_info.name,
group_info.size,
group_info.quota,
group_info.files))
if run_mode == 'collect':
store_group_quota(config, date_today, group_info_list)
logging.info('END')
sys.exit(0)
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
filename = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
error_msg = "Caught exception (%s): %s - %s (line: %s)" % \
(exc_type, str(e), filename, exc_tb.tb_lineno)
logging.error(error_msg)
sys.exit(1)
def create_group_info_list(file_system, input_file=None):
input_data = None
group_info_item_list = list()
if input_file:
if not os.path.isfile(input_file):
raise IOError(
"The input file does not exist or is not a file: %s" %
input_file)
with open(input_file, "r") as input_file:
input_data = input_file.read()
else:
check_path_exists(file_system)
output_list = list()
for group_name in get_user_groups():
output_list.append(
subprocess.check_output(
['sudo', LFS_BIN, 'quota', '-g', group_name,
file_system]).decode())
input_data = ''.join(output_list)
if not isinstance(input_data, str):
raise RuntimeError("Expected input data to be string, got: %s" %
type(input_data))
blocks = REGEX_QUOTA_PATTERN_BLOCK.findall(input_data)
for block in blocks:
lines = block.splitlines()
if len(lines) != 3:
raise RuntimeError("Invalid size for Block : %s" % block)
if not REGEX_QUOTA_PATTERN_INFO.match(lines[1]):
raise RuntimeError("Missing info line in block: %s" % block)
group_name = REGEX_QUOTA_PATTERN_HEADER.match(lines[0]).group(1)
data_result = REGEX_QUOTA_PATTERN_DATA.match(lines[2])
kbytes_used_raw = data_result.group(GroupQuotaCapturing.KBYTES_USED)
kbytes_quota = int(data_result.group(GroupQuotaCapturing.KBYTES_QUOTA))
files = int(data_result.group(GroupQuotaCapturing.FILES_COUNT))
# exclude '*' in kbytes field, if quota is exceeded!
if kbytes_used_raw[-1] == '*':
kbytes_used = int(kbytes_used_raw[:-1])
else:
kbytes_used = int(kbytes_used_raw)
bytes_used = kbytes_used * 1024
bytes_quota = kbytes_quota * 1024
group_info_item_list.append(
GroupInfoItem(group_name, bytes_used, bytes_quota, files))
logging.debug(group_info_item_list)
return group_info_item_list