def __setstate__(self, state):
Component.__setstate__(self, state)
self.msg_queue = state['msg_queue']
self.connected = False
self.decoder = LogMessageDecoder()
self.clearing_overflow = False
self.overflow_filename = None
self.overflow_file = None
self.max_queued = int(get_cdbwriter_config('max_queued_msgs', '-1'))
if self.max_queued <= 0:
logger.info("message queue set to unlimited.")
self.max_queued = None
else:
self.overflow_filename = get_cdbwriter_config('overflow_file', None)
if self.max_queued and (self.overflow_filename == None):
logger.warning("No file given to catch maximum messages. Setting queue size to unlimited.")
self.max_queued = None
if state.has_key('overflow') and self.max_queued:
self.overflow = state['overflow']
else:
self.overflow = False
def __init__ (self, *args, **kwargs):
"""Initialize a new ServiceLocator.
All arguments are passed to the component constructor.
"""
Component.__init__(self, *args, **kwargs)
self.services = ServiceDict()
def __init__ (self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.process_groups = ProcessGroupDict()
self.all_nodes = set()
self.running_nodes = set()
self.down_nodes = set()
self.queue_assignments = {}
self.node_order = {}
try:
self.configure(cluster_hostfile)
except:
self.logger.error("unable to load hostfile")
self.queue_assignments["default"] = set(self.all_nodes)
self.alloc_only_nodes = {} # nodename:starttime
self.cleaning_processes = []
#keep track of which jobs still have hosts being cleaned
self.cleaning_host_count = {} # jobid:count
self.locations_by_jobid = {} #jobid:[locations]
self.jobid_to_user = {} #jobid:username
self.alloc_timeout = int(get_cluster_system_config("allocation_timeout", 300))
self.logger.info("allocation timeout set to %d seconds." % self.alloc_timeout)
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.event_manager = ComponentProxy("event-manager")
self.bqsim = ComponentProxy("queue-manager")
self.powmon_logger = None
self.total_cost = 0.0
self.time_power_list =[{"unixtime":0, "power":0, "count":0, "utilization":0}]
def __setstate__(self, state):
Component.__setstate__(self, state)
self.reservations = state['reservations']
if 'active' in state:
self.active = state['active']
else:
self.active = True
self.id_gen = IncrID()
self.id_gen.set(state['next_res_id'])
global bgsched_id_gen
bgsched_id_gen = self.id_gen
self.cycle_id_gen = IncrID()
self.cycle_id_gen.set(state['next_cycle_id'])
global bgsched_cycle_id_gen
bgsched_cycle_id_gen = self.cycle_id_gen
self.queues = QueueDict()
self.jobs = JobDict()
self.started_jobs = {}
self.sync_state = Cobalt.Util.FailureMode("Foreign Data Sync")
self.get_current_time = time.time
if state.has_key('msg_queue'):
dbwriter.msg_queue = state['msg_queue']
if state.has_key('overflow') and (dbwriter.max_queued != None):
dbwriter.overflow = state['overflow']
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.resources = ResourceDict()
self.process_groups = ProcessGroupDict()
self.process_groups.item_cls = BBProcessGroup
self.queue_assignments = {}
self.queue_assignments["default"] = sets.Set(self.resources)
def __init__ (self, *args, **kwargs):
"""Initialize a new BaseForker.
All arguments are passed to the component constructor.
"""
Component.__init__(self, *args, **kwargs)
self.children = {}
self.id_gen = IncrID()
def __init__ (self, *args, **kwargs):
"""Initialize a new ServiceLocator.
All arguments are passed to the component constructor.
"""
Component.__init__(self, *args, **kwargs)
self.ignore = []
self.lastwait = 0
self.pgroups = ProcessGroupDict()
self.zombie_mpi = {}
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.COMP_QUEUE_MANAGER = "queue-manager"
self.COMP_SYSTEM = "system"
self.reservations = ReservationDict()
self.queues = QueueDict(self.COMP_QUEUE_MANAGER)
self.jobs = JobDict(self.COMP_QUEUE_MANAGER)
self.started_jobs = {}
self.sync_state = Cobalt.Util.FailureMode("Foreign Data Sync")
self.active = True
self.get_current_time = time.time
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.event_list = [{"unixtime": 0}]
self.time_stamp = 0
self.finished = False
self.bgsched = Sim_bg_Sched(**kwargs)
self.csched = Sim_Cluster_Sched()
self.go_next = True
def __init__ (self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self._partitions = PartitionDict()
self._managed_partitions = set()
self.process_groups = BGProcessGroupDict()
self.node_card_cache = dict()
self._partitions_lock = thread.allocate_lock()
self.pending_diags = dict()
self.failed_diags = list()
self.bridge_in_error = False
self.cached_partitions = None
self.offline_partitions = []
def __init__(self, *args, **kwargs):
logger.debug(
"heckle: System: init ... %s ... &&&&&&&&&&&&&&&&&&&&&&&&&&&&& I am here as well &&&&&&&&&&&&&&&&&&&&&&&&&"
% threading.current_thread().getName()
)
Component.__init__(self, *args, **kwargs)
self.process_groups = ProcessGroupDict()
self.process_groups.item_cls = HeckleProcessGroup
self.resources = ResourceDict()
self.queue_assignments["default"] = self.resources.keys()
print "\n\n\n\n"
print "Queue assignments are: %s" % self.queue_assignments
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.event_list = [{'unixtime':0}]
self.time_stamp = 0
self.finished = False
self.bgsched = Sim_bg_Sched(**kwargs)
#inhibit coscheduling and cluster simulation feature before bgsched.py makes change
#self.csched = Sim_Cluster_Sched()
self.go_next = True
def __init__ (self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.process_groups = ProcessGroupDict()
self.pending_diags = dict()
self.failed_diags = list()
self.all_nodes = sets.Set()
self.running_nodes = sets.Set()
self.down_nodes = sets.Set()
self.queue_assignments = {}
self.node_order = {}
try:
self.configure(CP.get("cluster_system", "hostfile"))
except:
self.logger.error("unable to load hostfile")
self.queue_assignments["default"] = sets.Set(self.all_nodes)
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.event_list = [{'unixtime':0}]
self.time_stamp = 0
self.finished = False
self.bgsched = Sim_bg_Sched(**kwargs)
self.csched = Sim_Cluster_Sched()
self.powermonitor = PowerMonitor()
self.go_next = True
# last scheduling time
self.last_schedule_time = 0
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.reservations = ReservationDict()
self.queues = QueueDict()
self.jobs = JobDict()
self.started_jobs = {}
self.sync_state = Cobalt.Util.FailureMode("Foreign Data Sync")
self.active = True
self.get_current_time = time.time
self.id_gen = IncrID()
global bgsched_id_gen
bgsched_id_gen = self.id_gen
self.cycle_id_gen = IncrID()
global bgsched_cycle_id_gen
bgsched_cycle_id_gen = self.cycle_id_gen
def __getstate__(self):
state = {}
state.update(Component.__getstate__(self))
state.update({
'cdbwriter_version': 1,
'msg_queue': self.msg_queue,
'overflow': self.overflow})
return state
def __getstate__(self):
state = {}
state.update(Component.__getstate__(self))
state.update({
"cluster_base_version": 1,
"queue_assignments": self.queue_assignments,
"down_nodes": self.down_nodes })
return state
def __getstate__(self):
state = {}
state.update(Component.__getstate__(self))
state.update({
'cdbwriter_version': 1,
'msg_queue': self.msg_queue,
'overflow': self.overflow
})
return state
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.COMP_QUEUE_MANAGER = "queue-manager"
self.COMP_SYSTEM = "system"
self.reservations = ReservationDict()
self.queues = QueueDict(self.COMP_QUEUE_MANAGER)
self.jobs = JobDict(self.COMP_QUEUE_MANAGER)
self.started_jobs = {}
self.sync_state = Cobalt.Util.FailureMode("Foreign Data Sync")
self.active = True
self.get_current_time = time.time
self.id_gen = IncrID()
global bgsched_id_gen
bgsched_id_gen = self.id_gen
self.cycle_id_gen = IncrID()
global bgsched_cycle_id_gen
bgsched_cycle_id_gen = self.cycle_id_gen
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.least_item = int(get_histm_config('least_item', 10)) # tunable
self.lastDays = int(get_histm_config("last_days", 60)) # tunable
self.jobinfo_file = get_histm_config("jobinfo_file", "jobinfo.hist")
self.jobinfo_script = get_histm_config("jobinfo_script", "jobinfo.py")
self.fraction = float(get_histm_config("fraction", 0.8))
self.job_dict = {} #historical job dictionary
self.project_set = set([]) #distinct project names of historical jobs
self.user_set = set([]) #distinct user names of historical jobs
self.pair_set = set([]) #distinct (user, project) pair
self.Ap_dict_proj = {} #dictionary of walltime adjusting parameters by project name
self.Ap_dict_user = {} #dictionary of walltime adjusting parameters by user name
self.Ap_dict_paired = {} #dictionary of walltime adjusting parameters by double key (user, project)
self.update_Ap_Dict()
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.least_item = int(get_histm_config('least_item', 10)) # tunable
self.lastDays = int(get_histm_config("last_days", 60)) # tunable
self.jobinfo_file = get_histm_config("jobinfo_file", "jobinfo.hist")
self.jobinfo_script = get_histm_config("jobinfo_script", "jobinfo.py")
self.fraction = float(get_histm_config("fraction", 0.8))
self.minimum_ap = float(get_histm_config("minimum_ap", 0.5))
self.job_dict = {} #historical job dictionary
self.project_set = set([]) #distinct project names of historical jobs
self.user_set = set([]) #distinct user names of historical jobs
self.pair_set = set([]) #distinct (user, project) pair
self.Ap_dict_proj = {} #dictionary of walltime adjusting parameters by project name
self.Ap_dict_user = {} #dictionary of walltime adjusting parameters by user name
self.Ap_dict_paired = {} #dictionary of walltime adjusting parameters by double key (user, project)
self.update_Ap_Dict()
def __getstate__(self):
state = {}
state.update(Component.__getstate__(self))
state.update({
'sched_version':1,
'reservations':self.reservations,
'active':self.active,
'next_res_id':self.id_gen.idnum+1,
'next_cycle_id':self.cycle_id_gen.idnum+1,
'msg_queue': dbwriter.msg_queue,
'overflow': dbwriter.overflow})
return state
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.sync_state = Cobalt.Util.FailureMode("Foreign Data Sync")
self.connected = False
self.msg_queue = []
self.decoder = LogMessageDecoder()
self.overflow = False
self.overflow_filename = None
self.overflow_file = None
self.clearing_overflow = False
self.max_queued = int(get_cdbwriter_config('max_queued_msgs', '-1'))
if self.max_queued <= 0:
logger.info("message queue set to unlimited.")
self.max_queued = None
self.overflow_filename = get_cdbwriter_config('overflow_file', None)
if self.overflow_filename == None:
logger.warning("No file given to catch maximum messages. Setting queue size to unlimited.")
self.max_queued = None
def __getstate__(self):
state = {}
state.update(Component.__getstate__(self))
state.update({
'sched_version': 1,
'reservations': self.reservations,
'active': self.active,
'next_res_id': self.id_gen.idnum + 1,
'next_cycle_id': self.cycle_id_gen.idnum + 1,
'msg_queue': dbwriter.msg_queue,
'overflow': dbwriter.overflow
})
return state
def __setstate__(self, state):
Component.__setstate__(self, state)
self.queue_assignments = state["queue_assignments"]
self.down_nodes = state["down_nodes"]
self.process_groups = ProcessGroupDict()
self.all_nodes = set()
self.running_nodes = set()
self.node_order = {}
try:
self.configure(cluster_hostfile)
except:
self.logger.error("unable to load hostfile")
self.alloc_only_nodes = {} # nodename:starttime
if not state.has_key("cleaning_processes"):
self.cleaning_processes = []
self.cleaning_host_count = {} # jobid:count
self.locations_by_jobid = {} #jobid:[locations]
self.jobid_to_user = {} #jobid:username
self.alloc_timeout = int(get_cluster_system_config("allocation_timeout", 300))
self.logger.info("allocation timeout set to %d seconds." % self.alloc_timeout)
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.event_list = [{'unixtime': 0}]
self.time_stamp = 0
self.finished = False
###
# samnickolay
self.jobs_queue_time_utilizations = {}
self.utilization_records = []
self.next_slowdown_threshold_time_step = None
self.high_priority_nodes = 0
self.low_priority_nodes = 0
# samnickolay
###
# dwang:
print("[dw_evsim] simu_name: %s. " % kwargs.get("name"))
print("[dw_evsim] simu_times: %d. " % kwargs.get("times"))
print("[dw_evsim] checkpoint: %s. " % kwargs.get("checkpoint"))
#
print("[dw_evsim] checkp_dsize: %s. " % kwargs.get("checkp_dsize"))
print("[dw_evsim] checkp_bw_write: %s. " %
kwargs.get("checkp_w_bandwidth"))
print("[dw_evsim] checkp_bw_read: %s. " %
kwargs.get("checkp_r_bandwidth"))
print("[dw_evsim] checkp_interval: %s. " %
kwargs.get("checkp_t_internval"))
# dwang
self.bgsched = Sim_bg_Sched(**kwargs)
#self.csched = Sim_Cluster_Sched()
self.mmon = metricmon()
self.go_next = True
def save_me(self):
'''Automatic method for saving off the component statefile
'''
Component.save(self)
def __setstate__(self, state):
Component.__setstate__(self, state)
def save_me(self):
'''Automatically write statefiles.'''
Component.save(self)
def __getstate__(self):
state = {}
state.update(Component.__getstate__(self))
state.update({
'slp_version': 1})
return state
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.event_manager = ComponentProxy("event-manager")
self.bqsim = ComponentProxy("queue-manager")
self.powmon_logger = None
import os
import socket
import threading
import xmlrpclib
import Cobalt.Server
from Cobalt.Server import find_intended_location, XMLRPCServer
from Cobalt.Components.base import Component
c = Component()
class TestFindIntendedLocation (object):
def setup (self):
assert not os.path.exists("testfile")
def teardown (self):
try:
os.remove("testfile")
except OSError:
assert not os.path.exists("testfile")
def test_nofile (self):
component = Component()
location = find_intended_location(component, config_files=["testfile"])
assert location == ("", 0)
def test_file_without_def (self):
testfile = open("testfile", "w")
print >> testfile, "[components]"
print >> testfile, "someothercomponent=https://localhost:8080"
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
def test_nofile (self):
component = Component()
location = find_intended_location(component, config_files=["testfile"])
assert location == ("", 0)
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.process_groups = ProcessGroupDict()
self.process_groups.item_cls = HeckleProcessGroup
self.queue_assignments["default"] = self.get_resources()
self.hacky_forbidden_nodes = [] #This is a temporary fix for the forbidden nodes issue
def __init__ (self, **kwargs):
Component.__init__(self, **kwargs)
self.jobs = JobDict()
def save_me(self):
'''Automatic method for saving off the component statefile
'''
Component.save(self)
def __getstate__(self):
state = {}
state.update(Component.__getstate__(self))
# state.update({
# "cluster_simulator_version": 1 })
return state
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.event_manager = ComponentProxy("event-manager")
self.bqsim = ComponentProxy("queue-manager")
self.mmon_logger = None
def __getstate__(self):
state = {}
state.update(Component.__getstate__(self))
# state.update({
# "cluster_simulator_version": 1 })
return state
def save_me(self):
Component.save(self)
def __init__ (self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.count = 0