class ClusterConfiguration(QconfObject):
""" This class encapsulates UGE cluster configuration object. """
#: Object version.
VERSION = '1.0'
#: Object name key.
NAME_KEY = None
#: Object keys that must be provided by user.
USER_PROVIDED_KEYS = []
#: Default values for required data keys for the global configuration.
#: Value for execd_spool_dir key will depend on $SGE_ROOT and $SGE_CELL.
REQUIRED_GLOBAL_DATA_DEFAULTS = {
'execd_spool_dir': 'SGE_ROOT/SGE_CELL/spool',
'mailer': '/bin/mail',
'xterm': '/usr/bin/xterm',
'load_sensor': None,
'prolog': None,
'epilog': None,
'shell_start_mode': 'unix_behavior',
'login_shells': ['sh', 'bash', 'ksh', 'csh', 'tcsh'],
'min_uid': 0,
'min_gid': 0,
'user_lists': None,
'xuser_lists': None,
'projects': None,
'xprojects': None,
'default_jc': None,
'enforce_jc': False,
'enforce_project': False,
'enforce_user': '******',
'load_report_time': '00:00:40',
'max_unheard': '00:04:00',
'reschedule_unknown': '00:00:00',
'loglevel': 'log_warning',
'administrator_mail': None,
'set_token_cmd': None,
'pag_cmd': None,
'token_extend_time': None,
'shepherd_cmd': None,
'qmaster_params': None,
'execd_params': ['KEEP_ACTIVE=ERROR'],
'reporting_params': {
'accounting': True,
'reporting': False,
'flush_time': '00:00:13',
'joblog': False,
'sharelog': '00:00:00'
},
'finished_jobs': 0,
'gid_range': '20000-20100',
'qlogin_command': 'builtin',
'qlogin_daemon': 'builtin',
'rlogin_command': 'builtin',
'rlogin_daemon': 'builtin',
'rsh_command': 'builtin',
'rsh_daemon': 'builtin',
'max_aj_instances': 2000,
'max_aj_tasks': 75000,
'max_u_jobs': 0,
'max_jobs': 0,
'max_advance_reservations': 0,
'auto_user_oticket': 0,
'auto_user_fshare': 0,
'auto_user_default_project': None,
'auto_user_delete_time': 86400,
'delegated_file_staging': False,
'reprioritize': 0,
'jsv_url': None,
'jsv_allowed_mod': ['ac', 'h', 'i', 'e', 'o', 'j', 'M', 'N', 'p', 'w'],
'cgroups_params': {
'cgroup_path': None,
'cpuset': False,
'mount': False,
'freezer': False,
'freeze_pe_tasks': False,
'killing': False,
'forced_numa': False,
'h_vmem_limit': False,
'm_mem_free_hard': False,
'm_mem_free_soft': False,
'min_memory_limit': 0
},
'lost_job_timeout': '00:00:00',
'enable_lost_job_reschedule': False,
}
#: Default values for required data keys for the host configuration.
REQUIRED_HOST_DATA_DEFAULTS = {
'mailer': '/bin/mail',
'xterm': '/usr/bin/xterm',
}
BOOL_KEY_MAP = QconfObject.get_bool_key_map(REQUIRED_GLOBAL_DATA_DEFAULTS)
INT_KEY_MAP = QconfObject.get_int_key_map(REQUIRED_GLOBAL_DATA_DEFAULTS)
FLOAT_KEY_MAP = QconfObject.get_float_key_map(
REQUIRED_GLOBAL_DATA_DEFAULTS)
DEFAULT_LIST_DELIMITER = ','
LIST_KEY_MAP = {
'login_shells': ',',
'user_lists': ',',
'xuser_lists': ',',
'projects': ',',
'xprojects': ',',
'qmaster_params': ',',
'execd_params': ',',
'jsv_allowed_mod': ',',
}
DEFAULT_DICT_DELIMITER = ' '
DICT_KEY_MAP = {
'reporting_params': ' ',
'cgroups_params': ' ',
}
UGE_CASE_SENSITIVE_KEYS = {}
def __init__(self,
name='global',
data=None,
metadata=None,
json_string=None):
"""
Class constructor.
:param name: Configuration name (default: 'global').
:type name: str
:param data: Configuration data. If provided, it will override corresponding data from JSON string representation.
:type data: dict
:param metadata: Configuration metadata. If provided, it will override corresponding metadata from JSON string representation.
:type metadata: dict
:param json_string: Configuration JSON string representation.
:type json_string: str
:raises: **InvalidArgument** - in case metadata is not a dictionary, JSON string is not valid, or it does not contain dictionary representing a ClusterConfiguration object.
"""
QconfObject.__init__(self,
name=name,
data=data,
metadata=metadata,
json_string=json_string)
if not self.name:
self.name = self.get_name_from_data()
if not self.name:
self.name = 'global'
def get_name_from_data(self):
for (key, value) in self.data.items():
if key.startswith('#'):
return key[1:-1] # remove comment and ending column characters
return None
def get_required_data_defaults(self):
if self.name == 'global':
return self.REQUIRED_GLOBAL_DATA_DEFAULTS
else:
return self.REQUIRED_HOST_DATA_DEFAULTS
def get_tmp_file(self):
tmp_dir_path = tempfile.mkdtemp()
tmp_file_path = os.path.join(tmp_dir_path, self.name)
tmp_file = open(tmp_file_path, 'w')
return (tmp_file, tmp_file_path, tmp_dir_path)
class SchedulerConfiguration(QconfObject):
""" This class encapsulates UGE scheduler configuration object. """
#: Object version.
VERSION = '1.0'
#: Object name key.
NAME_KEY = None
#: Object keys that must be provided by user.
USER_PROVIDED_KEYS = []
#: Default values for required data keys.
REQUIRED_DATA_DEFAULTS = {
'algorithm':
'default',
'schedule_interval':
'0:0:15',
'maxujobs':
0,
'queue_sort_method':
'load',
'job_load_adjustments':
'np_load_avg=0.50',
'load_adjustment_decay_time':
'0:7:30',
'load_formula':
'np_load_avg',
'schedd_job_info':
False,
'flush_submit_sec':
1,
'flush_finish_sec':
1,
'params':
None,
'reprioritize_interval':
'0:0:0',
'halftime':
168,
'usage_weight_list':
['wallclock=0.000000', 'cpu=1.000000', 'mem=0.000000', 'io=0.000000'],
'compensation_factor':
5.0,
'weight_user':
0.25,
'weight_project':
0.25,
'weight_department':
0.25,
'weight_job':
0.25,
'weight_tickets_functional':
0,
'weight_tickets_share':
0,
'share_override_tickets':
True,
'share_functional_shares':
True,
'max_functional_jobs_to_schedule':
200,
'report_pjob_tickets':
True,
'max_pending_tasks_per_job':
50,
'halflife_decay_list':
None,
'policy_hierarchy':
'OFS',
'weight_ticket':
0.01,
'weight_waiting_time':
0.0,
'weight_deadline':
3600000.0,
'weight_urgency':
0.1,
'weight_priority':
1.0,
'fair_urgency_list':
None,
'max_reservation':
0,
'default_duration':
float('inf'),
'backfilling':
'ON',
'prioritize_preemptees':
False,
'preemptees_keep_resources':
False,
'max_preemptees':
0,
'preemption_distance':
'00:15:00',
'preemption_priority_adjustments':
None,
}
BOOL_KEY_MAP = QconfObject.get_bool_key_map(REQUIRED_DATA_DEFAULTS)
INT_KEY_MAP = QconfObject.get_int_key_map(REQUIRED_DATA_DEFAULTS)
FLOAT_KEY_MAP = QconfObject.get_float_key_map(REQUIRED_DATA_DEFAULTS)
DEFAULT_LIST_DELIMITER = ','
LIST_KEY_MAP = {
'job_load_adjustments': ',',
'fair_urgency_list': ',',
}
DEFAULT_DICT_DELIMITER = ','
DICT_KEY_MAP = {
'usage_weight_list': ',',
'halflife_decay_list': ':',
}
UGE_CASE_SENSITIVE_KEYS = {
'schedd_job_info': string.lower,
}
def __init__(self, data=None, metadata=None, json_string=None):
"""
Class constructor.
:param data: Configuration data. If provided, it will override corresponding data from JSON string representation.
:type data: dict
:param metadata: Configuration metadata. If provided, it will override corresponding metadata from JSON string representation.
:type metadata: dict
:param json_string: Configuration JSON string representation.
:type json_string: str
:raises: **InvalidArgument** - in case metadata is not a dictionary, JSON string is not valid, or it does not contain dictionary representing a SchedulerConfiguration object.
"""
QconfObject.__init__(self,
data=data,
metadata=metadata,
json_string=json_string)
class ResourceQuotaSet(QconfObject):
""" This class encapsulates UGE resource quota set object. """
#: Object version.
VERSION = '1.0'
#: Object name key.
NAME_KEY = 'name'
#: Object keys that must be provided by set.
USER_PROVIDED_KEYS = ['name']
#: Default values for required data keys.
REQUIRED_DATA_DEFAULTS = {
'description': None,
'enabled': False,
'limit': ['to slots=0'],
}
BOOL_KEY_MAP = QconfObject.get_bool_key_map(REQUIRED_DATA_DEFAULTS)
DEFAULT_LIST_DELIMITER = ','
LIST_KEY_MAP = {
'limit': ',',
}
def __init__(self, name=None, data=None, metadata=None, json_string=None):
"""
Class constructor.
:param name: Resource quota set name. If provided, it will override set name from data or JSON string parameters ('name' key).
:type name: str
:param data: Resource quota set data. If provided, it will override corresponding data from set JSON string representation.
:type data: dict
:param metadata: Resource quota set metadata. If provided, it will override corresponding metadata from set JSON string representation.
:type metadata: dict
:param json_string: Resource quota set JSON string representation.
:type json_string: str
:raises: **InvalidArgument** - in case metadata is not a dictionary, JSON string is not valid, or it does not contain dictionary representing an ResourceQuotaList object.
"""
QconfObject.__init__(self,
name=name,
data=data,
metadata=metadata,
json_string=json_string)
def to_uge(self):
"""
Converts object to string acceptable as input for UGE qconf command.
:returns: Object's UGE-formatted string.
"""
lines = '{\n'
for key in ['name', 'description', 'enabled']:
value = self.data.get(key)
lines += '%s %s\n' % (key, self.py_to_uge(key, value))
limits = self.data.get('limit')
for limit in limits:
lines += 'limit %s\n' % (limit)
lines += '}\n'
return lines
def py_to_uge(self, key, value):
for (uge_value, py_value) in self.UGE_PYTHON_OBJECT_MAP.items():
if value == py_value and type(value) == type(py_value):
return uge_value
return value
def to_dict(self, input_string):
lines = input_string.split('\n')
object_data = {}
for line in lines:
if not line:
continue
if line.startswith('{'):
continue
if line.startswith('}'):
continue
key_value = line.split()
key = key_value[0]
value = line.replace(key, '').strip()
if key == 'limit':
limits = object_data.get(key, [])
limits.append(value)
object_data[key] = limits
else:
object_data[key] = self.uge_to_py(key, value)
return object_data
def uge_to_py(self, key, value):
uppercase_value = value.upper()
for (uge_value, py_value) in self.UGE_PYTHON_OBJECT_MAP.items():
if uge_value == uppercase_value:
return py_value
return value
class ShareTree(QconfDictList):
""" This class encapsulates UGE share tree object. """
#: Object version.
VERSION = '1.0'
#: Key that designates start of an object in a list
FIRST_KEY = 'id'
#: Object name key.
NAME_KEY = None
#: Object keys that must be provided by user.
USER_PROVIDED_KEYS = ['name']
#: Default values for required dictionary data keys.
REQUIRED_DATA_DEFAULTS = {
'id': 0,
'name': None,
'type': 1,
'shares': 0,
'childnodes': None,
}
BOOL_KEY_MAP = QconfObject.get_bool_key_map(REQUIRED_DATA_DEFAULTS)
INT_KEY_MAP = QconfObject.get_int_key_map(REQUIRED_DATA_DEFAULTS)
FLOAT_KEY_MAP = QconfObject.get_float_key_map(REQUIRED_DATA_DEFAULTS)
DEFAULT_LIST_DELIMITER = ','
def __init__(self, data=None, metadata=None, json_string=None):
"""
Class constructor.
:param data: Configuration data. If provided, it will override corresponding data from JSON string representation.
:type data: dict
:param metadata: Configuration metadata. If provided, it will override corresponding metadata from JSON string representation.
:type metadata: dict
:param json_string: Configuration JSON string representation.
:type json_string: str
:raises: **InvalidArgument** - in case metadata is not a dictionary, JSON string is not valid, or it does not represent a ShareTree object.
"""
QconfDictList.__init__(self,
data=data,
metadata=metadata,
json_string=json_string)
def to_uge(self):
"""
Converts object to string acceptable as input for UGE qconf command.
:returns: Object's UGE-formatted string.
"""
lines = ''
for d in self.data:
for key in ['id', 'name', 'type', 'shares', 'childnodes']:
value = d.get(key)
lines += '%s%s%s\n' % (key, self.KEY_VALUE_DELIMITER,
self.py_to_uge(key, value))
return lines