def __init__(self, *args, **kwargs):
"""Initialize a new ServiceLocator.
All arguments are passed to the component constructor.
"""
Component.__init__(self, *args, **kwargs)
self.services = ServiceDict()
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.event_list = [{'unixtime': 0}]
self.time_stamp = 0
self.finished = False
# dwang:
print("[dw_evsim] simu_name: %s. " % kwargs.get("name"))
print("[dw_evsim] simu_times: %d. " % kwargs.get("times"))
print("[dw_evsim] checkpoint: %s. " % kwargs.get("checkpoint"))
#
print("[dw_evsim] checkp_dsize: %s. " % kwargs.get("checkp_dsize"))
print("[dw_evsim] checkp_bw_write: %s. " %
kwargs.get("checkp_w_bandwidth"))
print("[dw_evsim] checkp_bw_read: %s. " %
kwargs.get("checkp_r_bandwidth"))
print("[dw_evsim] checkp_interval: %s. " %
kwargs.get("checkp_t_internval"))
# dwang
self.bgsched = Sim_bg_Sched(**kwargs)
#self.csched = Sim_Cluster_Sched()
self.mmon = metricmon()
self.go_next = True
def __init__ (self, *args, **kwargs):
"""Initialize a new ServiceLocator.
All arguments are passed to the component constructor.
"""
Component.__init__(self, *args, **kwargs)
self.services = ServiceDict()
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.process_groups = ProcessGroupDict()
self.process_groups.item_cls = HeckleProcessGroup
self.queue_assignments["default"] = self.get_resources()
self.hacky_forbidden_nodes = [
] #This is a temporary fix for the forbidden nodes issue
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.resources = ResourceDict()
self.process_groups = ProcessGroupDict()
self.process_groups.item_cls = BBProcessGroup
self.queue_assignments = {}
self.queue_assignments["default"] = sets.Set(self.resources)
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.sync_state = Cobalt.Util.FailureMode("Foreign Data Sync")
self.connected = False
self.msg_queue = []
self.decoder = LogMessageDecoder()
self.overflow = False
self.overflow_filename = None
self.overflow_file = None
self.clearing_overflow = False
self.max_queued = int(get_cdbwriter_config('max_queued_msgs', '-1'))
if self.max_queued <= 0:
logger.info("message queue set to unlimited.")
self.max_queued = None
self.overflow_filename = get_cdbwriter_config(
'overflow_file', None)
if self.overflow_filename == None:
logger.warning(
"No file given to catch maximum messages. Setting queue size to unlimited."
)
self.max_queued = None
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.event_manager = ComponentProxy("event-manager")
self.bqsim = ComponentProxy("queue-manager")
self.powmon_logger = None
self.total_cost = 0.0
self.time_power_list =[{"unixtime":0, "power":0, "count":0, "utilization":0}]
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.event_list = [{'unixtime':0}]
self.time_stamp = 0
self.finished = False
###
# samnickolay
self.jobs_queue_time_utilizations = {}
self.utilization_records = []
self.next_slowdown_threshold_time_step = None
self.high_priority_nodes = 0
self.low_priority_nodes = 0
# samnickolay
###
# dwang:
print("[dw_evsim] simu_name: %s. " %kwargs.get("name"))
print("[dw_evsim] simu_times: %d. " %kwargs.get("times"))
print("[dw_evsim] checkpoint: %s. " %kwargs.get("checkpoint"))
#
print("[dw_evsim] checkp_dsize: %s. " %kwargs.get("checkp_dsize"))
print("[dw_evsim] checkp_bw_write: %s. " %kwargs.get("checkp_w_bandwidth"))
print("[dw_evsim] checkp_bw_read: %s. " %kwargs.get("checkp_r_bandwidth"))
print("[dw_evsim] checkp_interval: %s. " %kwargs.get("checkp_t_internval"))
# dwang
self.bgsched = Sim_bg_Sched(**kwargs)
#self.csched = Sim_Cluster_Sched()
self.mmon = metricmon()
self.go_next = True
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.resources = ResourceDict()
self.process_groups = ProcessGroupDict()
self.process_groups.item_cls = BBProcessGroup
self.queue_assignments = {}
self.queue_assignments["default"] = sets.Set(self.resources)
def __init__ (self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.process_groups = ProcessGroupDict()
self.all_nodes = set()
self.running_nodes = set()
self.down_nodes = set()
self.queue_assignments = {}
self.node_order = {}
try:
self.configure(cluster_hostfile)
except:
self.logger.error("unable to load hostfile")
self.queue_assignments["default"] = set(self.all_nodes)
self.alloc_only_nodes = {} # nodename:starttime
self.cleaning_processes = []
#keep track of which jobs still have hosts being cleaned
self.cleaning_host_count = {} # jobid:count
self.locations_by_jobid = {} #jobid:[locations]
self.jobid_to_user = {} #jobid:username
self.alloc_timeout = int(get_cluster_system_config("allocation_timeout", 300))
self.logger.info("allocation timeout set to %d seconds." % self.alloc_timeout)
def __init__(self, *args, **kwargs):
"""Initialize a new BaseForker.
All arguments are passed to the component constructor.
"""
Component.__init__(self, *args, **kwargs)
self.children = {}
self.id_gen = IncrID()
def __init__ (self, *args, **kwargs):
"""Initialize a new BaseForker.
All arguments are passed to the component constructor.
"""
Component.__init__(self, *args, **kwargs)
self.children = {}
self.id_gen = IncrID()
def __init__(self, *args, **kwargs):
"""Initialize a new ServiceLocator.
All arguments are passed to the component constructor.
"""
Component.__init__(self, *args, **kwargs)
self.ignore = []
self.lastwait = 0
self.pgroups = ProcessGroupDict()
self.zombie_mpi = {}
def __init__ (self, *args, **kwargs):
"""Initialize a new ServiceLocator.
All arguments are passed to the component constructor.
"""
Component.__init__(self, *args, **kwargs)
self.ignore = []
self.lastwait = 0
self.pgroups = ProcessGroupDict()
self.zombie_mpi = {}
def __init__(self, *args, **kwargs):
logger.debug(
"heckle: System: init ... %s ... &&&&&&&&&&&&&&&&&&&&&&&&&&&&& I am here as well &&&&&&&&&&&&&&&&&&&&&&&&&"
% threading.current_thread().getName())
Component.__init__(self, *args, **kwargs)
self.process_groups = ProcessGroupDict()
self.process_groups.item_cls = HeckleProcessGroup
self.resources = ResourceDict()
self.queue_assignments["default"] = self.resources.keys()
print "\n\n\n\n"
print "Queue assignments are: %s" % self.queue_assignments
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.COMP_QUEUE_MANAGER = "queue-manager"
self.COMP_SYSTEM = "system"
self.reservations = ReservationDict()
self.queues = QueueDict(self.COMP_QUEUE_MANAGER)
self.jobs = JobDict(self.COMP_QUEUE_MANAGER)
self.started_jobs = {}
self.sync_state = Cobalt.Util.FailureMode("Foreign Data Sync")
self.active = True
self.get_current_time = time.time
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.event_list = [{'unixtime': 0}]
self.time_stamp = 0
self.finished = False
self.bgsched = Sim_bg_Sched(**kwargs)
self.csched = Sim_Cluster_Sched()
self.go_next = True
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.COMP_QUEUE_MANAGER = "queue-manager"
self.COMP_SYSTEM = "system"
self.reservations = ReservationDict()
self.queues = QueueDict(self.COMP_QUEUE_MANAGER)
self.jobs = JobDict(self.COMP_QUEUE_MANAGER)
self.started_jobs = {}
self.sync_state = Cobalt.Util.FailureMode("Foreign Data Sync")
self.active = True
self.get_current_time = time.time
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.event_list = [{"unixtime": 0}]
self.time_stamp = 0
self.finished = False
self.bgsched = Sim_bg_Sched(**kwargs)
self.csched = Sim_Cluster_Sched()
self.go_next = True
def __init__(self, *args, **kwargs):
logger.debug(
"heckle: System: init ... %s ... &&&&&&&&&&&&&&&&&&&&&&&&&&&&& I am here as well &&&&&&&&&&&&&&&&&&&&&&&&&"
% threading.current_thread().getName()
)
Component.__init__(self, *args, **kwargs)
self.process_groups = ProcessGroupDict()
self.process_groups.item_cls = HeckleProcessGroup
self.resources = ResourceDict()
self.queue_assignments["default"] = self.resources.keys()
print "\n\n\n\n"
print "Queue assignments are: %s" % self.queue_assignments
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self._partitions = PartitionDict()
self._managed_partitions = set()
self.process_groups = BGProcessGroupDict()
self.node_card_cache = dict()
self._partitions_lock = thread.allocate_lock()
self.pending_diags = dict()
self.failed_diags = list()
self.bridge_in_error = False
self.cached_partitions = None
self.offline_partitions = []
def __init__ (self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self._partitions = PartitionDict()
self._managed_partitions = set()
self.process_groups = BGProcessGroupDict()
self.node_card_cache = dict()
self._partitions_lock = thread.allocate_lock()
self.pending_diags = dict()
self.failed_diags = list()
self.bridge_in_error = False
self.cached_partitions = None
self.offline_partitions = []
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.event_list = [{'unixtime': 0}]
self.time_stamp = 0
self.finished = False
self.bgsched = Sim_bg_Sched(**kwargs)
#inhibit coscheduling and cluster simulation feature before bgsched.py makes change
#self.csched = Sim_Cluster_Sched()
self.go_next = True
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.event_list = [{'unixtime':0}]
self.time_stamp = 0
self.finished = False
self.bgsched = Sim_bg_Sched(**kwargs)
#inhibit coscheduling and cluster simulation feature before bgsched.py makes change
#self.csched = Sim_Cluster_Sched()
self.go_next = True
def __init__ (self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.process_groups = ProcessGroupDict()
self.pending_diags = dict()
self.failed_diags = list()
self.all_nodes = sets.Set()
self.running_nodes = sets.Set()
self.down_nodes = sets.Set()
self.queue_assignments = {}
self.node_order = {}
try:
self.configure(CP.get("cluster_system", "hostfile"))
except:
self.logger.error("unable to load hostfile")
self.queue_assignments["default"] = sets.Set(self.all_nodes)
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.process_groups = ProcessGroupDict()
self.pending_diags = dict()
self.failed_diags = list()
self.all_nodes = sets.Set()
self.running_nodes = sets.Set()
self.down_nodes = sets.Set()
self.queue_assignments = {}
self.node_order = {}
try:
self.configure(CP.get("cluster_system", "hostfile"))
except:
self.logger.error("unable to load hostfile")
self.queue_assignments["default"] = sets.Set(self.all_nodes)
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.event_list = [{'unixtime':0}]
self.time_stamp = 0
self.finished = False
self.bgsched = Sim_bg_Sched(**kwargs)
self.csched = Sim_Cluster_Sched()
self.powermonitor = PowerMonitor()
self.go_next = True
# last scheduling time
self.last_schedule_time = 0
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.reservations = ReservationDict()
self.queues = QueueDict()
self.jobs = JobDict()
self.started_jobs = {}
self.sync_state = Cobalt.Util.FailureMode("Foreign Data Sync")
self.active = True
self.get_current_time = time.time
self.id_gen = IncrID()
global bgsched_id_gen
bgsched_id_gen = self.id_gen
self.cycle_id_gen = IncrID()
global bgsched_cycle_id_gen
bgsched_cycle_id_gen = self.cycle_id_gen
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.reservations = ReservationDict()
self.queues = QueueDict()
self.jobs = JobDict()
self.started_jobs = {}
self.sync_state = Cobalt.Util.FailureMode("Foreign Data Sync")
self.active = True
self.get_current_time = time.time
self.id_gen = IncrID()
global bgsched_id_gen
bgsched_id_gen = self.id_gen
self.cycle_id_gen = IncrID()
global bgsched_cycle_id_gen
bgsched_cycle_id_gen = self.cycle_id_gen
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.least_item = int(get_histm_config('least_item', 10)) # tunable
self.lastDays = int(get_histm_config("last_days", 60)) # tunable
self.jobinfo_file = get_histm_config("jobinfo_file", "jobinfo.hist")
self.jobinfo_script = get_histm_config("jobinfo_script", "jobinfo.py")
self.fraction = float(get_histm_config("fraction", 0.8))
self.job_dict = {} #historical job dictionary
self.project_set = set([]) #distinct project names of historical jobs
self.user_set = set([]) #distinct user names of historical jobs
self.pair_set = set([]) #distinct (user, project) pair
self.Ap_dict_proj = {} #dictionary of walltime adjusting parameters by project name
self.Ap_dict_user = {} #dictionary of walltime adjusting parameters by user name
self.Ap_dict_paired = {} #dictionary of walltime adjusting parameters by double key (user, project)
self.update_Ap_Dict()
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.least_item = int(get_histm_config('least_item', 10)) # tunable
self.lastDays = int(get_histm_config("last_days", 60)) # tunable
self.jobinfo_file = get_histm_config("jobinfo_file", "jobinfo.hist")
self.jobinfo_script = get_histm_config("jobinfo_script", "jobinfo.py")
self.fraction = float(get_histm_config("fraction", 0.8))
self.minimum_ap = float(get_histm_config("minimum_ap", 0.5))
self.job_dict = {} #historical job dictionary
self.project_set = set([]) #distinct project names of historical jobs
self.user_set = set([]) #distinct user names of historical jobs
self.pair_set = set([]) #distinct (user, project) pair
self.Ap_dict_proj = {} #dictionary of walltime adjusting parameters by project name
self.Ap_dict_user = {} #dictionary of walltime adjusting parameters by user name
self.Ap_dict_paired = {} #dictionary of walltime adjusting parameters by double key (user, project)
self.update_Ap_Dict()
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.sync_state = Cobalt.Util.FailureMode("Foreign Data Sync")
self.connected = False
self.msg_queue = []
self.decoder = LogMessageDecoder()
self.overflow = False
self.overflow_filename = None
self.overflow_file = None
self.clearing_overflow = False
self.max_queued = int(get_cdbwriter_config('max_queued_msgs', '-1'))
if self.max_queued <= 0:
logger.info("message queue set to unlimited.")
self.max_queued = None
self.overflow_filename = get_cdbwriter_config('overflow_file', None)
if self.overflow_filename == None:
logger.warning("No file given to catch maximum messages. Setting queue size to unlimited.")
self.max_queued = None
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.process_groups = ProcessGroupDict()
self.process_groups.item_cls = HeckleProcessGroup
self.queue_assignments["default"] = self.get_resources()
self.hacky_forbidden_nodes = [] #This is a temporary fix for the forbidden nodes issue
def __init__ (self, **kwargs):
Component.__init__(self, **kwargs)
self.jobs = JobDict()
def __init__ (self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.count = 0
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.event_manager = ComponentProxy("event-manager")
self.bqsim = ComponentProxy("queue-manager")
self.powmon_logger = None
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.event_manager = ComponentProxy("event-manager")
self.bqsim = ComponentProxy("queue-manager")
self.mmon_logger = None