def __setstate__(self, state): Cobalt.Util.fix_set(state) if hasattr(state, 'register_component') and state['register_component']: self._registered_component=True Cobalt.Proxy.register_component(self) else: self._registered_component=False self.logger = logging.getLogger("%s %s" % (self.implementation, self.name)) self._component_lock = threading.Lock() self._component_lock_acquired_time = None self.statistics = Statistics()
def __setstate__(self, state): sys.setrecursionlimit(5000) self._managed_partitions = state['managed_partitions'] self._partitions = PartitionDict() self.process_groups = BGProcessGroupDict() self.process_groups.item_cls = BGProcessGroup self.node_card_cache = dict() self._partitions_lock = thread.allocate_lock() self.pending_diags = dict() self.failed_diags = list() self.diag_pids = dict() self.pending_script_waits = sets.Set() self.bridge_in_error = False self.cached_partitions = None self.offline_partitions = [] self.configure() if 'partition_flags' in state: for pname, flags in state['partition_flags'].items(): if pname in self._partitions: self._partitions[pname].scheduled = flags[0] self._partitions[pname].functional = flags[1] self._partitions[pname].queue = flags[2] else: logger.info("Partition %s is no longer defined" % pname) self.update_relatives() # initiate the process before starting any threads thread.start_new_thread(self.update_partition_state, tuple()) self.lock = threading.Lock() self.statistics = Statistics()
def __setstate__(self, state): self.reservations = state['reservations'] if 'active' in state: self.active = state['active'] else: self.active = True self.id_gen = IncrID() self.id_gen.set(state['next_res_id']) global bgsched_id_gen bgsched_id_gen = self.id_gen self.cycle_id_gen = IncrID() self.cycle_id_gen.set(state['next_cycle_id']) global bgsched_cycle_id_gen bgsched_cycle_id_gen = self.cycle_id_gen self.queues = QueueDict(self.COMP_QUEUE_MANAGER) self.jobs = JobDict(self.COMP_QUEUE_MANAGER) self.started_jobs = {} self.sync_state = Cobalt.Util.FailureMode("Foreign Data Sync") self.get_current_time = time.time self.lock = threading.Lock() self.statistics = Statistics() if state.has_key('msg_queue'): dbwriter.msg_queue = state['msg_queue'] if state.has_key('overflow') and (dbwriter.max_queued != None): dbwriter.overflow = state['overflow']
def __setstate__(self, state): self._managed_partitions = state['managed_partitions'] self.config_file = state['config_file'] self._partitions = PartitionDict() self.process_groups = BGProcessGroupDict() self.process_groups.item_cls = BGSimProcessGroup self.node_card_cache = dict() self._partitions_lock = thread.allocate_lock() self.failed_components = sets.Set() self.pending_diags = dict() self.failed_diags = list() self.bridge_in_error = False self.cached_partitions = None self.offline_partitions = [] if self.config_file is not None: self.configure(self.config_file) if 'partition_flags' in state: for pname, flags in state['partition_flags'].items(): if pname in self._partitions: self._partitions[pname].scheduled = flags[0] self._partitions[pname].functional = flags[1] self._partitions[pname].queue = flags[2] else: logger.info("Partition %s is no longer defined" % pname) self.update_relatives() self.lock = threading.Lock() self.statistics = Statistics()
def __init__ (self, **kwargs): """Initialize a new component. Keyword arguments: statefile -- file in which to save state automatically """ self.statefile = kwargs.get("statefile", None) if kwargs.get("register", True): self._registered_component=True Cobalt.Proxy.register_component(self) else: self._registered_component=False self.logger = logging.getLogger("%s %s" % (self.implementation, self.name)) self._component_lock = threading.Lock() self._component_lock_acquired_time = None self.statistics = Statistics()
def __setstate__(self, state): Cobalt.Util.fix_set(state) if hasattr(state, "register_component") and state["register_component"]: self._registered_component = True Cobalt.Proxy.register_component(self) else: self._registered_component = False self.logger = logging.getLogger("%s %s" % (self.implementation, self.name)) self._component_lock = threading.Lock() self._component_lock_acquired_time = None self.statistics = Statistics()
def __setstate__(self, state): Cobalt.Util.fix_set(state) if hasattr(state, 'register_component') and state['register_component']: self._registered_component=True Cobalt.Proxy.register_component(self) else: self._registered_component=False self.logger = logging.getLogger("%s %s" % (self.implementation, self.name)) self._component_lock = threading.Lock() self._component_lock_acquired_time = None self.statistics = Statistics() self.logger.info("%s:%s component executing on %s", self.name, self.implementation, socket.gethostname())
def __setstate__(self, state): self.reservations = state['reservations'] if 'active' in state: self.active = state['active'] else: self.active = True self.queues = QueueDict(self.COMP_QUEUE_MANAGER) self.jobs = JobDict(self.COMP_QUEUE_MANAGER) self.started_jobs = {} self.sync_state = Cobalt.Util.FailureMode("Foreign Data Sync") self.get_current_time = time.time self.lock = threading.Lock() self.statistics = Statistics()
def __setstate__(self, state): self.queue_assignments = state["queue_assignments"] self.down_nodes = state["down_nodes"] self.process_groups = ProcessGroupDict() self.pending_diags = dict() self.failed_diags = list() self.all_nodes = sets.Set() self.running_nodes = sets.Set() self.node_order = {} try: self.configure(CP.get("cluster_system", "hostfile")) except: self.logger.error("unable to load hostfile") self.lock = threading.Lock() self.statistics = Statistics()
def __init__(self, **kwargs): """Initialize a new component. Keyword arguments: statefile -- file in which to save state automatically """ self.statefile = kwargs.get("statefile", None) if kwargs.get("register", True): self._registered_component = True Cobalt.Proxy.register_component(self) else: self._registered_component = False self.logger = logging.getLogger("%s %s" % (self.implementation, self.name)) self._component_lock = threading.Lock() self._component_lock_acquired_time = None self.statistics = Statistics()
def __setstate__(self, state): Cobalt.Util.fix_set(state) self.queue_assignments = state["queue_assignments"] self.down_nodes = state["down_nodes"] self.process_groups = ProcessGroupDict() self.all_nodes = set() self.running_nodes = set() self.node_order = {} try: self.configure(cluster_hostfile) except: self.logger.error("unable to load hostfile") self.lock = threading.Lock() self.statistics = Statistics() self.alloc_only_nodes = {} # nodename:starttime if not state.has_key("cleaning_processes"): self.cleaning_processes = [] self.cleaning_host_count = {} # jobid:count self.locations_by_jobid = {} #jobid:[locations] self.jobid_to_user = {} #jobid:username self.alloc_timeout = int(get_cluster_system_config("allocation_timeout", 300)) self.logger.info("allocation timeout set to %d seconds." % self.alloc_timeout)
class Component(object): """Base component. Intended to be served as an instance by Cobalt.Component.XMLRPCServer >>> server = Cobalt.Component.XMLRPCServer(location, keyfile) >>> component = Cobalt.Component.Component() >>> server.serve_instance(component) Class attributes: name -- logical component name (e.g., "queue-manager", "process-manager") implementation -- implementation identifier (e.g., "BlueGene/L", "BlueGene/P") Methods: save -- pickle the component to a file do_tasks -- perform automatic tasks for the component """ name = "component" implementation = "generic" def __init__(self, **kwargs): """Initialize a new component. Keyword arguments: statefile -- file in which to save state automatically """ self.statefile = kwargs.get("statefile", None) if kwargs.get("register", True): self._registered_component = True Cobalt.Proxy.register_component(self) else: self._registered_component = False self.logger = logging.getLogger("%s %s" % (self.implementation, self.name)) self._component_lock = threading.Lock() self._component_lock_acquired_time = None self.statistics = Statistics() def __getstate__(self): state = {} return {"base_component_version": 1, "register_component": self._registered_component} def __setstate__(self, state): Cobalt.Util.fix_set(state) if hasattr(state, "register_component") and state["register_component"]: self._registered_component = True Cobalt.Proxy.register_component(self) else: self._registered_component = False self.logger = logging.getLogger("%s %s" % (self.implementation, self.name)) self._component_lock = threading.Lock() self._component_lock_acquired_time = None self.statistics = Statistics() def component_lock_acquire(self): entry_time = time.time() self._component_lock.acquire() self._component_lock_acquired_time = time.time() self.statistics.add_value("component_lock_wait", self._component_lock_acquired_time - entry_time) def component_lock_release(self): self.statistics.add_value("component_lock_held", time.time() - self._component_lock_acquired_time) self._component_lock_acquired_time = None self._component_lock.release() def save(self, statefile=None): """Pickle the component. Arguments: statefile -- use this file, rather than component.statefile """ statefile = statefile or self.statefile if statefile: temp_statefile = statefile + ".temp" data = cPickle.dumps(self) try: fd = file(temp_statefile, "wb") fd.write(data) fd.close() except IOError, e: self.logger.error("statefile failure : %s" % e) return str(e) else: os.rename(temp_statefile, statefile) return "state saved to file: %s" % statefile
class Component(object): """Base component. Intended to be served as an instance by Cobalt.Component.XMLRPCServer >>> server = Cobalt.Component.XMLRPCServer(location, keyfile) >>> component = Cobalt.Component.Component() >>> server.serve_instance(component) Class attributes: name -- logical component name (e.g., "queue-manager", "process-manager") implementation -- implementation identifier (e.g., "BlueGene/L", "BlueGene/P") Methods: save -- pickle the component to a file do_tasks -- perform automatic tasks for the component """ name = "component" implementation = "generic" def __init__(self, **kwargs): """Initialize a new component. Keyword arguments: statefile -- file in which to save state automatically """ self.statefile = kwargs.get("statefile", None) if kwargs.get("register", True): self._registered_component = True Cobalt.Proxy.register_component(self) else: self._registered_component = False self.logger = logging.getLogger("%s %s" % (self.implementation, self.name)) self._component_lock = threading.Lock() self._component_lock_acquired_time = None self.statistics = Statistics() def __getstate__(self): state = {} return { 'base_component_version': 1, 'register_component': self._registered_component } def __setstate__(self, state): Cobalt.Util.fix_set(state) if hasattr(state, 'register_component') and state['register_component']: self._registered_component = True Cobalt.Proxy.register_component(self) else: self._registered_component = False self.logger = logging.getLogger("%s %s" % (self.implementation, self.name)) self._component_lock = threading.Lock() self._component_lock_acquired_time = None self.statistics = Statistics() def component_lock_acquire(self): entry_time = time.time() self._component_lock.acquire() self._component_lock_acquired_time = time.time() self.statistics.add_value( 'component_lock_wait', self._component_lock_acquired_time - entry_time) def component_lock_release(self): self.statistics.add_value( 'component_lock_held', time.time() - self._component_lock_acquired_time) self._component_lock_acquired_time = None self._component_lock.release() def save(self, statefile=None): """Pickle the component. Arguments: statefile -- use this file, rather than component.statefile """ statefile = statefile or self.statefile if statefile: temp_statefile = statefile + ".temp" data = cPickle.dumps(self) try: fd = file(temp_statefile, "wb") fd.write(data) fd.close() except IOError, e: self.logger.error("statefile failure : %s" % e) return str(e) else: os.rename(temp_statefile, statefile) return "state saved to file: %s" % statefile