def process_single_task(self, task):
client = task['client']
category_list = task['category_list']
disk_usage_result_model = get_disk_usage(category_list, client)
for an_item in disk_usage_result_model.items:
for a_gauge in self.gauge_list:
a_gauge['metric'].labels(owner=task['owner'],
repo=task['repo'],
file_system=get_property_data(
an_item, "disk_usage.dir")).set(
get_property_data(
an_item,
a_gauge['category_id']))
def process_single_task(self, task):
client = task['client']
category_list = task['category_list']
model = self.request(category_list, client)
if model is None:
return
for an_item in model.items:
for a_gauge in self.gauge_list:
labels = {}
for a_label_config in self.gauge_label_config:
label_name = a_label_config['name']
label_type = a_label_config['type']
if label_type == "task":
config_key = a_label_config['config_key']
labels[label_name] = task[config_key]
elif label_type == "category":
category_id = a_label_config['category_id']
labels[label_name] = get_property_data(
an_item, category_id)
else:
raise Exception("label type not supported:",
label_type)
data = get_property_data(an_item, a_gauge['category_id'])
if 'value_extractor' in a_gauge:
value_extractor_config = a_gauge['value_extractor']
value_extractor_class = globals()[
value_extractor_config['class']]
value_extractor_arguments = value_extractor_config[
'arguments']
value_extractor = value_extractor_class(
**value_extractor_arguments)
value = value_extractor.extract(data)
else:
value = data
a_gauge['metric'].labels(**labels).set(value)
def get_sort_data(job_item, property_id):
data = get_property_data(job_item, property_id)
if property_id == 'llq.status':
status_map = {
'R': 0,
'C': 10,
'I': 100,
'RP': 200,
'H': 300,
}
status = data
return status_map.get(status, 500)
else:
return data
def process_single_task(self, task):
client = task['client']
category_list = task['category_list']
disk_space_result_model = get_disk_space(category_list, client)
for an_item in disk_space_result_model.items:
for a_gauge in self.gauge_list:
labels = {}
for a_label_config in self.gauge_label_config:
label_name = a_label_config['name']
label_type = a_label_config['type']
if label_type == "task":
config_key = a_label_config['config_key']
labels[label_name] = task[config_key]
elif label_type == "category":
category_id = a_label_config['category_id']
labels[label_name] = get_property_data(
an_item, category_id)
else:
raise Exception("label type not supported:",
label_type)
a_gauge['metric'].labels(**labels).set(
get_property_data(an_item, a_gauge['category_id']))
def show_detail(config_file, user_list, class_list, sort_keys, params):
"""
Query jobs in LoadLeveler and show in a detailed format.
"""
config = get_config(config_file)
if params is None:
params = ''
if user_list:
params += ' -u {user_list}'.format(user_list=" ".join(user_list))
if class_list:
params += ' -c {class_list}'.format(class_list=" ".join(class_list))
if sort_keys:
sort_keys = tuple(sort_keys.split(":"))
model_dict = get_llq_detail_query_response(config, params)
items = model_dict['items']
sort_query_response_items(items, sort_keys)
for an_item in items:
job_id = get_property_data(an_item, "llq.id")
job_class = get_property_data(an_item, "llq.class")
job_owner = get_property_data(an_item, "llq.owner")
job_script = get_property_data(an_item, "llq.job_script")
job_status = get_property_data(an_item, "llq.status")
job_queue_data = get_property_data(an_item, "llq.queue_date")
job_err = get_property_data(an_item, "llq.err")
job_out = get_property_data(an_item, "llq.out")
click.echo(
"""{job_id} {job_status} {job_class} {job_owner} {job_queue_data}
Script: {job_script}
Out: {job_out}
Err: {job_err}
""".format(job_id=click.style(job_id, bold=True),
job_class=click.style(job_class, fg='blue'),
job_owner=click.style(job_owner, fg='cyan'),
job_queue_data=click.style(job_queue_data.strftime("%m/%d %H:%M"),
fg='blue'),
job_script=job_script,
job_status=click.style(job_status, fg='yellow'),
job_err=job_err,
job_out=job_out))
def vierr(config_file, job_id):
"""
Edit error output file.
"""
config = get_config(config_file)
params = job_id
model_dict = get_llq_detail_query_response(config, params)
job_count = len(model_dict['items'])
if job_count == 0:
click.echo('There is no job.')
elif job_count > 1:
click.echo('There are more than one job.')
else:
an_item = model_dict['items'][0]
# job_script = get_property_data(an_item, "llq.job_script")
job_err = get_property_data(an_item, "llq.err")
# job_out = get_property_data(an_item, "llq.out")
click.edit(filename=job_err)
def vijob(config_file, file_type, job_id):
"""
Edit job script.
"""
config = get_config(config_file)
params = job_id
model_dict = get_llq_detail_query_response(config, params)
job_count = len(model_dict['items'])
if job_count == 0:
click.echo('There is no job.')
elif job_count > 1:
click.echo('There are more than one job.')
else:
an_item = model_dict['items'][0]
key = "llq.job_script"
if file_type == 'out':
key = 'llq.out'
elif file_type == 'err':
key = 'llq.err'
file_path = get_property_data(an_item, key)
click.edit(filename=file_path)
def get_sort_data(job_item, property_id):
data = get_property_data(job_item, property_id)
return data
def apply_filter(filter_name, detail, config_file):
"""
apply filter for loadleveler.
"""
config = get_config(config_file)
params = ''
sort_keys = None
query_model = get_llq_detail_query_response(config, params)
from loadleveler_client.plugins.filters import long_time_operation_job_filter, nwp_pd_long_time_upload_job_filter
filter_module_list = [
long_time_operation_job_filter, nwp_pd_long_time_upload_job_filter
]
def apply_filters(job_items):
results = []
for a_filter_module in filter_module_list:
a_filter_object = a_filter_module.create_filter()
cur_filter_name = a_filter_object['name']
a_filter = a_filter_object['filter']
target_job_items = a_filter.filter(job_items)
results.append({
'name': cur_filter_name,
'target_job_items': target_job_items
})
return results
filter_results = apply_filters(query_model['items'])
for a_filter_result in filter_results:
click.echo('{filter_name}:'.format(
filter_name=click.style(a_filter_result['name'], bold=True)))
max_class_length = 0
max_owner_length = 0
for an_item in a_filter_result['target_job_items']:
job_class = get_property_data(an_item, "llq.class")
if len(job_class) > max_class_length:
max_class_length = len(job_class)
job_owner = get_property_data(an_item, "llq.owner")
if len(job_owner) > max_owner_length:
max_owner_length = len(job_owner)
items = a_filter_result['target_job_items']
sort_query_response_items(items, sort_keys)
for an_item in items:
job_id = get_property_data(an_item, "llq.id")
job_class = get_property_data(an_item, "llq.class")
job_owner = get_property_data(an_item, "llq.owner")
job_script = get_property_data(an_item, "llq.job_script")
job_status = get_property_data(an_item, "llq.status")
job_queue_data = get_property_data(an_item, "llq.queue_date")
click.echo(
"{job_id} {job_status} {job_class} {job_owner} {job_queue_data} {job_script}"
.format(
job_id=click.style(job_id, bold=True),
job_class=click.style(
("{job_class: <" + str(max_class_length) +
"}").format(job_class=job_class),
fg='blue'),
job_owner=click.style(
("{job_owner: <" + str(max_owner_length) +
"}").format(job_owner=job_owner),
fg='cyan'),
job_queue_data=click.style(
job_queue_data.strftime("%m/%d %H:%M"), fg='blue'),
job_script=job_script,
job_status=click.style(
"{job_status: <2}".format(job_status=job_status),
fg='yellow'),
))
click.echo()
def query_user_llq(config, user_name, sort_keys, long=False):
model_dict = get_llq_detail_query_response(config)
max_class_length = 0
max_owner_length = 0
for an_item in model_dict['items']:
job_owner = get_property_data(an_item, "llq.owner")
if user_name not in job_owner:
continue
job_class = get_property_data(an_item, "llq.class")
if len(job_class) > max_class_length:
max_class_length = len(job_class)
if len(job_owner) > max_owner_length:
max_owner_length = len(job_owner)
items = model_dict['items']
sort_query_response_items(items, sort_keys)
for an_item in items:
job_id = get_property_data(an_item, "llq.id")
job_class = get_property_data(an_item, "llq.class")
job_owner = get_property_data(an_item, "llq.owner")
if user_name not in job_owner:
continue
job_script = get_property_data(an_item, "llq.job_script")
job_status = get_property_data(an_item, "llq.status")
job_queue_data = get_property_data(an_item, "llq.queue_date")
if long:
job_err = get_property_data(an_item, "llq.err")
job_out = get_property_data(an_item, "llq.out")
click.echo(
"""{job_id} {job_status} {job_class} {job_owner} {job_queue_data}
Script: {job_script}
Out: {job_out}
Err: {job_err}
""".format(job_id=click.style(job_id, bold=True),
job_class=click.style(job_class, fg='blue'),
job_owner=click.style(job_owner, fg='cyan'),
job_queue_data=job_queue_data.strftime("%m/%d %H:%M"),
job_script=job_script,
job_status=click.style(job_status, fg='yellow'),
job_err=job_err,
job_out=job_out))
else:
click.echo(
"{job_id} {job_status} {job_class} {job_owner} {job_queue_data} {job_script}"
.format(
job_id=click.style(job_id, bold=True),
job_class=click.style(
("{job_class: <" + str(max_class_length) +
"}").format(job_class=job_class),
fg='blue'),
job_owner=click.style(
("{job_owner: <" + str(max_owner_length) +
"}").format(job_owner=job_owner),
fg='cyan'),
job_queue_data=click.style(
job_queue_data.strftime("%m/%d %H:%M"), fg='blue'),
job_script=job_script,
job_status=click.style(
"{job_status: <2}".format(job_status=job_status),
fg='yellow'),
))
def query(config_file, user_list, class_list, sort_keys, params,
job_script_pattern):
"""
Query jobs in LoadLeveler and show in a simple format.
"""
config = get_config(config_file)
if params is None:
params = ''
if user_list:
params += ' -u {user_list}'.format(user_list=" ".join(user_list))
if class_list:
params += ' -c {class_list}'.format(class_list=" ".join(class_list))
if sort_keys:
sort_keys = tuple(sort_keys.split(":"))
model_dict = get_llq_detail_query_response(config, params)
max_class_length = 0
max_owner_length = 0
for an_item in model_dict['items']:
job_class = get_property_data(an_item, "llq.class")
if len(job_class) > max_class_length:
max_class_length = len(job_class)
job_owner = get_property_data(an_item, "llq.owner")
if len(job_owner) > max_owner_length:
max_owner_length = len(job_owner)
items = model_dict['items']
if job_script_pattern:
from loadleveler_client.plugins.filters import job_script_filter
a_filter_object = job_script_filter.create_filter(job_script_pattern)
a_filter = a_filter_object['filter']
items = a_filter.filter(items)
sort_query_response_items(items, sort_keys)
for an_item in items:
job_id = get_property_data(an_item, "llq.id")
job_class = get_property_data(an_item, "llq.class")
job_owner = get_property_data(an_item, "llq.owner")
job_script = get_property_data(an_item, "llq.job_script")
job_status = get_property_data(an_item, "llq.status")
job_queue_data = get_property_data(an_item, "llq.queue_date")
click.echo(
"{job_id} {job_status} {job_class} {job_owner} {job_queue_data} {job_script}"
.format(
job_id=click.style(job_id, bold=True),
job_class=click.style(
("{job_class: <" + str(max_class_length) +
"}").format(job_class=job_class),
fg='blue'),
job_owner=click.style(
("{job_owner: <" + str(max_owner_length) +
"}").format(job_owner=job_owner),
fg='cyan'),
job_queue_data=click.style(
job_queue_data.strftime("%m/%d %H:%M"), fg='blue'),
job_script=job_script,
job_status=click.style(
"{job_status: <2}".format(job_status=job_status),
fg='yellow'),
))