class ExecutionHost(QconfObject): """ This class encapsulates UGE execution host object. """ #: Object version. VERSION = '1.0' #: Object name key. NAME_KEY = 'hostname' #: Object keys that must be provided by user. USER_PROVIDED_KEYS = ['hostname'] #: Default values for required data keys. REQUIRED_DATA_DEFAULTS = { 'load_scaling': None, 'complex_values': None, 'user_lists': None, 'xuser_lists': None, 'projects': None, 'xprojects': None, 'usage_scaling': None, 'report_variables': None, 'license_constraints': None, 'license_oversubscription': None, } INT_KEY_MAP = QconfObject.get_int_key_map(REQUIRED_DATA_DEFAULTS) FLOAT_KEY_MAP = QconfObject.get_float_key_map(REQUIRED_DATA_DEFAULTS) DEFAULT_LIST_DELIMITER = ',' LIST_KEY_MAP = { 'complex_values': ',', 'user_lists': ',', 'xuser_lists': ',', 'projects': ',', 'xprojects': ',', 'report_variables': ',', } def __init__(self, name=None, data=None, metadata=None, json_string=None): """ Class constructor. :param name: Execution host name. If provided, it will override execution host name from data or JSON string parameters ('hostname' key). :type name: str :param data: Execution host data. If provided, it will override corresponding data from execution host JSON string representation. :type data: dict :param metadata: Execution host metadata. If provided, it will override corresponding metadata from execution host JSON string representation. :type metadata: dict :param json_string: Execution host JSON string representation. :type json_string: str :raises: **InvalidArgument** - in case metadata is not a dictionary, JSON string is not valid, or it does not contain dictionary representing an ExecutionHost object. """ QconfObject.__init__(self, name=name, data=data, metadata=metadata, json_string=json_string)
class ParallelEnvironment(QconfObject): """ This class encapsulates UGE parallel environment object. """ #: Object version. VERSION = '1.0' #: Object name key. NAME_KEY = 'pe_name' #: Object keys that must be provided by user. USER_PROVIDED_KEYS = ['pe_name'] #: Default values for required data keys. REQUIRED_DATA_DEFAULTS = { 'slots': 0, 'user_lists': None, 'xuser_lists': None, 'start_proc_args': None, 'stop_proc_args': None, 'allocation_rule': '$pe_slots', 'control_slaves': False, 'job_is_first_task': True, 'urgency_slots': 'min', 'accounting_summary': False, 'daemon_forks_slaves': False, 'master_forks_slaves': False, } INT_KEY_MAP = QconfObject.get_int_key_map(REQUIRED_DATA_DEFAULTS) FLOAT_KEY_MAP = QconfObject.get_float_key_map(REQUIRED_DATA_DEFAULTS) DEFAULT_LIST_DELIMITER = ' ' LIST_KEY_MAP = { 'user_lists': ' ', 'xuser_lists': ' ', } def __init__(self, name=None, data=None, metadata=None, json_string=None): """ Class constructor. :param name: PE name. If provided, it will override PE name from data or JSON string parameters ('pe_name' key). :type name: str :param data: PE data. If provided, it will override corresponding data from PE JSON string representation. :type data: dict :param metadata: PE metadata. If provided, it will override corresponding metadata from PE JSON string representation. :type metadata: dict :param json_string: PE JSON string representation. :type json_string: str :raises: **InvalidArgument** - in case metadata is not a dictionary, JSON string is not valid, or it does not contain dictionary representing a PE object. """ QconfObject.__init__(self, name=name, data=data, metadata=metadata, json_string=json_string)
class CheckpointingEnvironment(QconfObject): """ This class encapsulates UGE checkpointing environment object. """ #: Object version. VERSION = '1.0' #: Object name key. NAME_KEY = 'ckpt_name' #: Object keys that must be provided by user. USER_PROVIDED_KEYS = ['ckpt_name'] #: Default values for required data keys. REQUIRED_DATA_DEFAULTS = { 'interface': 'userdefined', 'ckpt_command': None, 'migr_command': None, 'restart_command': None, 'clean_command': None, 'ckpt_dir': '/tmp', 'signal': None, 'when': 'sx' } INT_KEY_MAP = QconfObject.get_int_key_map(REQUIRED_DATA_DEFAULTS) FLOAT_KEY_MAP = QconfObject.get_float_key_map(REQUIRED_DATA_DEFAULTS) DEFAULT_LIST_DELIMITER = ' ' LIST_KEY_MAP = {} def __init__(self, name=None, data=None, metadata=None, json_string=None): """ Class constructor. :param name: Checkpointing environment name. If provided, it will override environment name from data or JSON string parameters ('ckpt_name' key). :type name: str :param data: Checkpointing environment data. If provided, it will override corresponding data from environment JSON string representation. :type data: dict :param metadata: Checkpointing environment metadata. If provided, it will override corresponding metadata from environment JSON string representation. :type metadata: dict :param json_string: Checkpointing environment JSON string representation. :type json_string: str :raises: **InvalidArgument** - in case metadata is not a dictionary, JSON string is not valid, or it does not contain dictionary representing a PE object. """ QconfObject.__init__(self, name=name, data=data, metadata=metadata, json_string=json_string)
class Project(QconfObject): """ This class encapsulates UGE project object. """ #: Object version. VERSION = '1.0' #: Object name key. NAME_KEY = 'name' #: Object keys that must be provided by user. USER_PROVIDED_KEYS = ['name'] #: Default values for required data keys. REQUIRED_DATA_DEFAULTS = { 'oticket': 0, 'fshare': 0, 'acl': None, 'xacl': None, } INT_KEY_MAP = QconfObject.get_int_key_map(REQUIRED_DATA_DEFAULTS) FLOAT_KEY_MAP = QconfObject.get_float_key_map(REQUIRED_DATA_DEFAULTS) DEFAULT_LIST_DELIMITER = ' ' LIST_KEY_MAP = { 'acl': ' ', 'xacl': ' ', } def __init__(self, name=None, data=None, metadata=None, json_string=None): """ Class constructor. :param name: Project name. If provided, it will override project name from data or JSON string parameters ('name' key). :type name: str :param data: Project data. If provided, it will override corresponding data from project JSON string representation. :type data: dict :param metadata: Project metadata. If provided, it will override corresponding metadata from project JSON string representation. :type metadata: dict :param json_string: Project JSON string representation. :type json_string: str :raises: **InvalidArgument** - in case metadata is not a dictionary, JSON string is not valid, or it does not contain dictionary representing an Project object. """ QconfObject.__init__(self, name=name, data=data, metadata=metadata, json_string=json_string)
class Calendar(QconfObject): """ This class encapsulates UGE calendar object. """ #: Object version. VERSION = '1.0' #: Object name key. NAME_KEY = 'calendar_name' #: Object keys that must be provided by user. USER_PROVIDED_KEYS = ['calendar_name'] #: Default values for required data keys. REQUIRED_DATA_DEFAULTS = { 'year': None, 'week': None, } INT_KEY_MAP = QconfObject.get_int_key_map(REQUIRED_DATA_DEFAULTS) FLOAT_KEY_MAP = QconfObject.get_float_key_map(REQUIRED_DATA_DEFAULTS) def __init__(self, name=None, data=None, metadata=None, json_string=None): """ Class constructor. :param name: Calendar name. If provided, it will override calendar name from data or JSON string parameters ('name' key). :type name: str :param data: Calendar data. If provided, it will override corresponding data from calendar JSON string representation. :type data: dict :param metadata: Calendar metadata. If provided, it will override corresponding metadata from calendar JSON string representation. :type metadata: dict :param json_string: Calendar JSON string representation. :type json_string: str :raises: **InvalidArgument** - in case metadata is not a dictionary, JSON string is not valid, or it does not contain dictionary representing an Calendar object. """ QconfObject.__init__(self, name=name, data=data, metadata=metadata, json_string=json_string)
class ClusterQueue(QconfObject): """ This class encapsulates UGE cluster queue object. """ #: Object version. VERSION = '2.0' #: Object name key. NAME_KEY = 'qname' #: Object keys that must be provided by user. USER_PROVIDED_KEYS = ['qname'] #: Default values for required data keys. REQUIRED_DATA_DEFAULTS = { 'hostlist' : None, 'seq_no' : 0, 'load_thresholds' : 'np_load_avg=1.75', 'suspend_thresholds' : None, 'nsuspend' : 1, 'suspend_interval' : '00:05:00', 'priority' : 0, 'min_cpu_interval' : '00:05:00', 'qtype' : 'BATCH INTERACTIVE', 'ckpt_list' : None, 'pe_list' : 'make', 'jc_list' : ['NO_JC','ANY_JC'], 'rerun' : False, 'rerun_limit' : 0, 'rerun_limit_action' : None, 'slots' : 1, 'tmpdir' : '/tmp', 'shell' : '/bin/sh', 'prolog' : None, 'epilog' : None, 'shell_start_mode' : 'unix_behavior', 'starter_method' : None, 'suspend_method' : None, 'resume_method' : None, 'terminate_method' : None, 'notify' : '00:00:60', 'owner_list' : None, 'user_lists' : None, 'xuser_lists' : None, 'subordinate_list' : None, 'complex_values' : None, 'projects' : None, 'xprojects' : None, 'calendar' : None, 'initial_state' : 'default', 's_rt' : float('inf'), 'h_rt' : float('inf'), 'd_rt' : float('inf'), 's_cpu' : float('inf'), 'h_cpu' : float('inf'), 's_fsize' : float('inf'), 'h_fsize' : float('inf'), 's_data' : float('inf'), 'h_data' : float('inf'), 's_stack' : float('inf'), 'h_stack' : float('inf'), 's_core' : float('inf'), 'h_core' : float('inf'), 's_rss' : float('inf'), 'h_rss' : float('inf'), 's_vmem' : float('inf'), 'h_vmem' : float('inf') } INT_KEY_MAP = QconfObject.get_int_key_map(REQUIRED_DATA_DEFAULTS) FLOAT_KEY_MAP = QconfObject.get_float_key_map(REQUIRED_DATA_DEFAULTS) DEFAULT_LIST_DELIMITER = ',' LIST_KEY_MAP = { 'slots' : ',', 'load_thresholds' : ',', 'suspend_thresholds' : ',', 'ckpt_list' : ',', 'pe_list' : ',', 'jc_list' : ',', 'owner_list' : ',', 'user_lists' : ',', 'xuser_lists' : ',', 'subordinate_list' : ',', 'complex_values' : ',', 'projects' : ',', 'xprojects' : ',', } def __init__(self, name=None, data=None, metadata=None, json_string=None): """ Class constructor. :param name: Queue name. If provided, it will override queue name from data or JSON string parameters ('qname' key). :type name: str :param data: Queue data. If provided, it will override corresponding data from queue JSON string representation. :type data: dict :param metadata: Queue metadata. If provided, it will override corresponding metadata from queue JSON string representation. :type metadata: dict :param json_string: Queue JSON string representation. :type json_string: str :raises: **InvalidArgument** - in case metadata is not a dictionary, JSON string is not valid, or it does not contain dictionary representing a ClusterQueue object. """ QconfObject.__init__(self, name=name, data=data, metadata=metadata, json_string=json_string)
class ClusterConfiguration(QconfObject): """ This class encapsulates UGE cluster configuration object. """ #: Object version. VERSION = '1.0' #: Object name key. NAME_KEY = None #: Object keys that must be provided by user. USER_PROVIDED_KEYS = [] #: Default values for required data keys for the global configuration. #: Value for execd_spool_dir key will depend on $SGE_ROOT and $SGE_CELL. REQUIRED_GLOBAL_DATA_DEFAULTS = { 'execd_spool_dir': 'SGE_ROOT/SGE_CELL/spool', 'mailer': '/bin/mail', 'xterm': '/usr/bin/xterm', 'load_sensor': None, 'prolog': None, 'epilog': None, 'shell_start_mode': 'unix_behavior', 'login_shells': ['sh', 'bash', 'ksh', 'csh', 'tcsh'], 'min_uid': 0, 'min_gid': 0, 'user_lists': None, 'xuser_lists': None, 'projects': None, 'xprojects': None, 'default_jc': None, 'enforce_jc': False, 'enforce_project': False, 'enforce_user': '******', 'load_report_time': '00:00:40', 'max_unheard': '00:04:00', 'reschedule_unknown': '00:00:00', 'loglevel': 'log_warning', 'administrator_mail': None, 'set_token_cmd': None, 'pag_cmd': None, 'token_extend_time': None, 'shepherd_cmd': None, 'qmaster_params': None, 'execd_params': ['KEEP_ACTIVE=ERROR'], 'reporting_params': { 'accounting': True, 'reporting': False, 'flush_time': '00:00:13', 'joblog': False, 'sharelog': '00:00:00' }, 'finished_jobs': 0, 'gid_range': '20000-20100', 'qlogin_command': 'builtin', 'qlogin_daemon': 'builtin', 'rlogin_command': 'builtin', 'rlogin_daemon': 'builtin', 'rsh_command': 'builtin', 'rsh_daemon': 'builtin', 'max_aj_instances': 2000, 'max_aj_tasks': 75000, 'max_u_jobs': 0, 'max_jobs': 0, 'max_advance_reservations': 0, 'auto_user_oticket': 0, 'auto_user_fshare': 0, 'auto_user_default_project': None, 'auto_user_delete_time': 86400, 'delegated_file_staging': False, 'reprioritize': 0, 'jsv_url': None, 'jsv_allowed_mod': ['ac', 'h', 'i', 'e', 'o', 'j', 'M', 'N', 'p', 'w'], 'cgroups_params': { 'cgroup_path': None, 'cpuset': False, 'mount': False, 'freezer': False, 'freeze_pe_tasks': False, 'killing': False, 'forced_numa': False, 'h_vmem_limit': False, 'm_mem_free_hard': False, 'm_mem_free_soft': False, 'min_memory_limit': 0 }, 'lost_job_timeout': '00:00:00', 'enable_lost_job_reschedule': False, } #: Default values for required data keys for the host configuration. REQUIRED_HOST_DATA_DEFAULTS = { 'mailer': '/bin/mail', 'xterm': '/usr/bin/xterm', } BOOL_KEY_MAP = QconfObject.get_bool_key_map(REQUIRED_GLOBAL_DATA_DEFAULTS) INT_KEY_MAP = QconfObject.get_int_key_map(REQUIRED_GLOBAL_DATA_DEFAULTS) FLOAT_KEY_MAP = QconfObject.get_float_key_map( REQUIRED_GLOBAL_DATA_DEFAULTS) DEFAULT_LIST_DELIMITER = ',' LIST_KEY_MAP = { 'login_shells': ',', 'user_lists': ',', 'xuser_lists': ',', 'projects': ',', 'xprojects': ',', 'qmaster_params': ',', 'execd_params': ',', 'jsv_allowed_mod': ',', } DEFAULT_DICT_DELIMITER = ' ' DICT_KEY_MAP = { 'reporting_params': ' ', 'cgroups_params': ' ', } UGE_CASE_SENSITIVE_KEYS = {} def __init__(self, name='global', data=None, metadata=None, json_string=None): """ Class constructor. :param name: Configuration name (default: 'global'). :type name: str :param data: Configuration data. If provided, it will override corresponding data from JSON string representation. :type data: dict :param metadata: Configuration metadata. If provided, it will override corresponding metadata from JSON string representation. :type metadata: dict :param json_string: Configuration JSON string representation. :type json_string: str :raises: **InvalidArgument** - in case metadata is not a dictionary, JSON string is not valid, or it does not contain dictionary representing a ClusterConfiguration object. """ QconfObject.__init__(self, name=name, data=data, metadata=metadata, json_string=json_string) if not self.name: self.name = self.get_name_from_data() if not self.name: self.name = 'global' def get_name_from_data(self): for (key, value) in self.data.items(): if key.startswith('#'): return key[1:-1] # remove comment and ending column characters return None def get_required_data_defaults(self): if self.name == 'global': return self.REQUIRED_GLOBAL_DATA_DEFAULTS else: return self.REQUIRED_HOST_DATA_DEFAULTS def get_tmp_file(self): tmp_dir_path = tempfile.mkdtemp() tmp_file_path = os.path.join(tmp_dir_path, self.name) tmp_file = open(tmp_file_path, 'w') return (tmp_file, tmp_file_path, tmp_dir_path)
class JobClass(QconfObject): """ This class encapsulates UGE job class object. """ #: Object version. VERSION = '2.0' #: Object name key. NAME_KEY = 'jcname' #: Object keys that must be provided by user. USER_PROVIDED_KEYS = ['jcname'] #: Default values for required data keys. REQUIRED_DATA_DEFAULTS = { 'variant_list': None, 'owner': None, 'user_lists': None, 'xuser_lists': None, 'A': '{+}UNSPECIFIED', 'a': '{+}UNSPECIFIED', 'ar': '{+}UNSPECIFIED', 'b': '{+}UNSPECIFIED', 'binding': '{+}UNSPECIFIED', 'c_interval': '{+}UNSPECIFIED', 'c_occasion': '{+}UNSPECIFIED', 'CMDNAME': '{+}UNSPECIFIED', 'CMDARG': '{+}UNSPECIFIED', 'ckpt': '{+}UNSPECIFIED', 'ac': '{+}UNSPECIFIED', 'cwd': '{+}UNSPECIFIED', 'dl': '{+}UNSPECIFIED', 'e': '{+}UNSPECIFIED', 'h': '{+}UNSPECIFIED', 'hold_jid': '{+}UNSPECIFIED', 'hold_jid_ad': '{+}UNSPECIFIED', 'i': '{+}UNSPECIFIED', 'j': '{+}UNSPECIFIED', 'js': '{+}UNSPECIFIED', 'l_hard': '{+}UNSPECIFIED', 'l_soft': '{+}UNSPECIFIED', 'masterl': '{+}UNSPECIFIED', 'm': '{+}UNSPECIFIED', 'mbind': '{+}UNSPECIFIED', 'M': '{+}UNSPECIFIED', 'masterq': '{+}UNSPECIFIED', 'N': '{+}UNSPECIFIED', 'notify': '{+}UNSPECIFIED', 'now': '{+}UNSPECIFIED', 'o': '{+}UNSPECIFIED', 'P': '{+}UNSPECIFIED', 'p': '{+}UNSPECIFIED', 'pe_name': '{+}UNSPECIFIED', 'pe_range': '{+}UNSPECIFIED', 'q_hard': '{+}UNSPECIFIED', 'q_soft': '{+}UNSPECIFIED', 'R': '{+}UNSPECIFIED', 'r': '{+}UNSPECIFIED', 'rou': '{+}UNSPECIFIED', 'S': '{+}UNSPECIFIED', 'shell': '{+}UNSPECIFIED', 't': '{+}UNSPECIFIED', 'tc': '{+}UNSPECIFIED', 'V': '{+}UNSPECIFIED', 'v': '{+}UNSPECIFIED', 'xd': '{+}UNSPECIFIED', } INT_KEY_MAP = QconfObject.get_int_key_map(REQUIRED_DATA_DEFAULTS) FLOAT_KEY_MAP = QconfObject.get_float_key_map(REQUIRED_DATA_DEFAULTS) DEFAULT_LIST_DELIMITER = ',' LIST_KEY_MAP = { 'variant_list': ',', 'user_lists': ',', 'xuser_lists': ',', } def __init__(self, name=None, data=None, metadata=None, json_string=None): """ Class constructor. :param name: Job class name. If provided, it will override job class name from data or JSON string parameters ('jcname' key). :type name: str :param data: Job class data. If provided, it will override corresponding data from job class JSON string representation. :type data: dict :param metadata: Job class metadata. If provided, it will override corresponding metadata from job class JSON string representation. :type metadata: dict :param json_string: Job class JSON string representation. :type json_string: str :raises: **InvalidArgument** - in case metadata is not a dictionary, JSON string is not valid, or it does not contain dictionary representing a JobClass object. """ QconfObject.__init__(self, name=name, data=data, metadata=metadata, json_string=json_string)
class SchedulerConfiguration(QconfObject): """ This class encapsulates UGE scheduler configuration object. """ #: Object version. VERSION = '1.0' #: Object name key. NAME_KEY = None #: Object keys that must be provided by user. USER_PROVIDED_KEYS = [] #: Default values for required data keys. REQUIRED_DATA_DEFAULTS = { 'algorithm': 'default', 'schedule_interval': '0:0:15', 'maxujobs': 0, 'queue_sort_method': 'load', 'job_load_adjustments': 'np_load_avg=0.50', 'load_adjustment_decay_time': '0:7:30', 'load_formula': 'np_load_avg', 'schedd_job_info': False, 'flush_submit_sec': 1, 'flush_finish_sec': 1, 'params': None, 'reprioritize_interval': '0:0:0', 'halftime': 168, 'usage_weight_list': ['wallclock=0.000000', 'cpu=1.000000', 'mem=0.000000', 'io=0.000000'], 'compensation_factor': 5.0, 'weight_user': 0.25, 'weight_project': 0.25, 'weight_department': 0.25, 'weight_job': 0.25, 'weight_tickets_functional': 0, 'weight_tickets_share': 0, 'share_override_tickets': True, 'share_functional_shares': True, 'max_functional_jobs_to_schedule': 200, 'report_pjob_tickets': True, 'max_pending_tasks_per_job': 50, 'halflife_decay_list': None, 'policy_hierarchy': 'OFS', 'weight_ticket': 0.01, 'weight_waiting_time': 0.0, 'weight_deadline': 3600000.0, 'weight_urgency': 0.1, 'weight_priority': 1.0, 'fair_urgency_list': None, 'max_reservation': 0, 'default_duration': float('inf'), 'backfilling': 'ON', 'prioritize_preemptees': False, 'preemptees_keep_resources': False, 'max_preemptees': 0, 'preemption_distance': '00:15:00', 'preemption_priority_adjustments': None, } BOOL_KEY_MAP = QconfObject.get_bool_key_map(REQUIRED_DATA_DEFAULTS) INT_KEY_MAP = QconfObject.get_int_key_map(REQUIRED_DATA_DEFAULTS) FLOAT_KEY_MAP = QconfObject.get_float_key_map(REQUIRED_DATA_DEFAULTS) DEFAULT_LIST_DELIMITER = ',' LIST_KEY_MAP = { 'job_load_adjustments': ',', 'fair_urgency_list': ',', } DEFAULT_DICT_DELIMITER = ',' DICT_KEY_MAP = { 'usage_weight_list': ',', 'halflife_decay_list': ':', } UGE_CASE_SENSITIVE_KEYS = { 'schedd_job_info': string.lower, } def __init__(self, data=None, metadata=None, json_string=None): """ Class constructor. :param data: Configuration data. If provided, it will override corresponding data from JSON string representation. :type data: dict :param metadata: Configuration metadata. If provided, it will override corresponding metadata from JSON string representation. :type metadata: dict :param json_string: Configuration JSON string representation. :type json_string: str :raises: **InvalidArgument** - in case metadata is not a dictionary, JSON string is not valid, or it does not contain dictionary representing a SchedulerConfiguration object. """ QconfObject.__init__(self, data=data, metadata=metadata, json_string=json_string)
class ShareTree(QconfDictList): """ This class encapsulates UGE share tree object. """ #: Object version. VERSION = '1.0' #: Key that designates start of an object in a list FIRST_KEY = 'id' #: Object name key. NAME_KEY = None #: Object keys that must be provided by user. USER_PROVIDED_KEYS = ['name'] #: Default values for required dictionary data keys. REQUIRED_DATA_DEFAULTS = { 'id': 0, 'name': None, 'type': 1, 'shares': 0, 'childnodes': None, } BOOL_KEY_MAP = QconfObject.get_bool_key_map(REQUIRED_DATA_DEFAULTS) INT_KEY_MAP = QconfObject.get_int_key_map(REQUIRED_DATA_DEFAULTS) FLOAT_KEY_MAP = QconfObject.get_float_key_map(REQUIRED_DATA_DEFAULTS) DEFAULT_LIST_DELIMITER = ',' def __init__(self, data=None, metadata=None, json_string=None): """ Class constructor. :param data: Configuration data. If provided, it will override corresponding data from JSON string representation. :type data: dict :param metadata: Configuration metadata. If provided, it will override corresponding metadata from JSON string representation. :type metadata: dict :param json_string: Configuration JSON string representation. :type json_string: str :raises: **InvalidArgument** - in case metadata is not a dictionary, JSON string is not valid, or it does not represent a ShareTree object. """ QconfDictList.__init__(self, data=data, metadata=metadata, json_string=json_string) def to_uge(self): """ Converts object to string acceptable as input for UGE qconf command. :returns: Object's UGE-formatted string. """ lines = '' for d in self.data: for key in ['id', 'name', 'type', 'shares', 'childnodes']: value = d.get(key) lines += '%s%s%s\n' % (key, self.KEY_VALUE_DELIMITER, self.py_to_uge(key, value)) return lines