def work_seek(self): try: while True: job = self.web_resource.queue.get(block=False) if job is None: raise Queue.Empty() yield self.send( self.queue_name_work, 'work', { "work_amount": job.length, "batchid": job.batchid, "jobid": job.jobid }) extradict = { "batchid": job.batchid, "jobid": job.jobid, "work_amount": job.length } cei_events.event("workproducer", "job_sent", extra=extradict) # This is an unfortunate hack to work around a memory leak in ion. # Some caches are only cleared after a received message is handled. # Since this process sends messages "spontaneously" -- triggered by a # LoopingCall -- we must manually clear the cache. self.message_client.workbench.manage_workbench_cache( 'Default Context') except Queue.Empty: return except Exception, e: # unhandled exceptions will terminate the LoopingCall log.error("Error adding work: %s", e, exc_info=True)
def work_seek(self): try: while True: job = self.web_resource.queue.get(block=False) if job is None: raise Queue.Empty() yield self.send(self.queue_name_work, 'work', {"work_amount":job.length, "batchid":job.batchid, "jobid":job.jobid}) extradict = {"batchid":job.batchid, "jobid":job.jobid, "work_amount":job.length} cei_events.event("workproducer", "job_sent", extra=extradict) # This is an unfortunate hack to work around a memory leak in ion. # Some caches are only cleared after a received message is handled. # Since this process sends messages "spontaneously" -- triggered by a # LoopingCall -- we must manually clear the cache. self.message_client.workbench.manage_workbench_cache('Default Context') except Queue.Empty: return except Exception,e: # unhandled exceptions will terminate the LoopingCall log.error("Error adding work: %s", e, exc_info=True)
def launch(self, deployable_type_id, launch_description, extravars=None): """Choose instance IDs for each instance desired, a launch ID and send appropriate message to Provisioner. Control API method, see the decision engine implementer's guide. @param deployable_type_id string identifier of the DP to launch @param launch_description See engine implementer's guide @param extravars Optional, see engine implementer's guide @retval tuple (launch_id, launch_description), see guide @exception Exception illegal input @exception Exception message not sent """ # right now we are sending some node-specific data in provisioner vars # (node_id at least) if len(launch_description) != 1: raise NotImplementedError("Only single-node launches are supported") launch_id = str(uuid.uuid4()) log.info("Request for DP '%s' is a new launch with id '%s'" % (deployable_type_id, launch_id)) new_instance_id_list = [] for group, item in launch_description.iteritems(): log.info(" - %s is %d %s from %s" % (group, item.num_instances, item.allocation_id, item.site)) if item.num_instances != 1: raise NotImplementedError("Only single-node launches are supported") for i in range(item.num_instances): new_instance_id = str(uuid.uuid4()) self.state.new_instance_launch(new_instance_id, launch_id, item.site, item.allocation_id) item.instance_ids.append(new_instance_id) new_instance_id_list.append(new_instance_id) vars_send = self.prov_vars.copy() if extravars: vars_send.update(extravars) # The node_id var is the reason only single-node launches are supported. # It could be instead added by the provisioner or something? It also # is complicated by the contextualization system. vars_send["node_id"] = new_instance_id_list[0] vars_send["heartbeat_dest"] = self.controller_name # hide passwords from logging hide_password = deepcopy(vars_send) if "cassandra_password" in hide_password: hide_password["cassandra_password"] = "******" if "broker_password" in hide_password: hide_password["broker_password"] = "******" log.debug("Launching with parameters:\n%s" % str(hide_password)) subscribers = (self.controller_name,) self.provisioner.provision(launch_id, deployable_type_id, launch_description, subscribers, vars=vars_send) extradict = {"launch_id": launch_id, "new_instance_ids": new_instance_id_list, "subscribers": subscribers} cei_events.event("controller", "new_launch", extra=extradict) return launch_id, launch_description
def test_event_write(self): self.log.debug("something") cei_events.event("unittest", "TRIAL1", self.log) self.log.debug("something-else") events = cei_events.events_from_file(self.logfilepath) assert len(events) == 1 assert events[0].source == "unittest" assert events[0].name == "TRIAL1"
def test_event_write(self): log.debug("something") cei_events.event("unittest", "TRIAL1", log) log.debug("something-else") events = cei_events.events_from_file(self.logfilepath) assert len(events) == 1 assert events[0].source == "unittest" assert events[0].name == "TRIAL1"
def query_one_site(self, site, nodes, driver=None): node_driver = driver or self.site_drivers[site] log.info('Querying site "%s"', site) nimboss_nodes = yield threads.deferToThread(node_driver.list_nodes) nimboss_nodes = dict((node.id, node) for node in nimboss_nodes) # note we are walking the nodes from datastore, NOT from nimboss for node in nodes: state = node['state'] if state < states.PENDING or state >= states.TERMINATED: continue nimboss_id = node.get('iaas_id') nimboss_node = nimboss_nodes.pop(nimboss_id, None) if not nimboss_node: # this state is unknown to underlying IaaS. What could have # happened? IaaS error? Recovery from loss of net to IaaS? # Or lazily-updated records. On EC2, there can be a short # window where pending instances are not included in query # response start_time = node.get('pending_timestamp') now = time.time() if start_time and (now - start_time) <= _IAAS_NODE_QUERY_WINDOW_SECONDS: log.debug('node %s: not in query of IaaS, but within '+ 'allowed startup window (%d seconds)', node['node_id'], _IAAS_NODE_QUERY_WINDOW_SECONDS) else: log.warn('node %s: in data store but unknown to IaaS. '+ 'Marking as terminated.', node['node_id']) node['state'] = states.FAILED node['state_desc'] = 'NODE_DISAPPEARED' launch = yield self.store.get_launch(node['launch_id']) if launch: yield self.store_and_notify([node], launch['subscribers']) else: nimboss_state = _NIMBOSS_STATE_MAP[nimboss_node.state] if nimboss_state > node['state']: #TODO nimboss could go backwards in state. node['state'] = nimboss_state update_node_ip_info(node, nimboss_node) if nimboss_state == states.STARTED: extradict = {'iaas_id': nimboss_id, 'node_id': node.get('node_id'), 'public_ip': node.get('public_ip'), 'private_ip': node.get('private_ip') } cei_events.event("provisioner", "node_started", extra=extradict) launch = yield self.store.get_launch(node['launch_id']) yield self.store_and_notify([node], launch['subscribers'])
def slc_init(self): cei_events.event("provisioner", "init_begin") try: store = self.spawn_args['store'] site_drivers = self.spawn_args['site_drivers'] context_client = self.spawn_args['context_client'] except KeyError,e: raise KeyError("Missing provisioner spawn_arg: " + str(e))
def slc_init(self): cei_events.event("provisioner", "init_begin") try: store = self.spawn_args['store'] site_drivers = self.spawn_args['site_drivers'] context_client = self.spawn_args['context_client'] except KeyError, e: raise KeyError("Missing provisioner spawn_arg: " + str(e))
def start(self): self.dashi.handle(self.work) extradict = {"queue_name_work": self.queue_name_work} cei_events.event("worker", "init_end", extra=extradict) try: self.dashi.consume() except KeyboardInterrupt: self.log.info("Caught terminate signal. Bye!")
def work(self, work_amount=0, batchid="", jobid=""): extradict = {"batchid": batchid, "jobid": jobid, "work_amount": work_amount} cei_events.event("worker", "job_begin", extra=extradict) self.log.info("WORK: sleeping for %d seconds ---" % work_amount) time.sleep(work_amount) cei_events.event("worker", "job_end", extra=extradict) return {"result": "work_complete"}
def slc_init(self): queue_name = self.spawn_args["queue_name_work"] self.workReceiver = WorkerReceiver(name=queue_name, label=__name__, scope=WorkerReceiver.SCOPE_SYSTEM, handler=self.receive) self.queue_name_work = self.workReceiver.xname extradict = {"queue_name_work":self.queue_name_work} cei_events.event("worker", "init_begin", extra=extradict) self.laterinitialized = False reactor.callLater(0, self.later_init)
def slc_init(self): queue_name = self.spawn_args["queue_name_work"] self.workReceiver = WorkerReceiver(name=queue_name, label=__name__, scope=WorkerReceiver.SCOPE_SYSTEM, handler=self.receive) self.queue_name_work = self.workReceiver.xname extradict = {"queue_name_work": self.queue_name_work} cei_events.event("worker", "init_begin", extra=extradict) self.laterinitialized = False reactor.callLater(0, self.later_init)
def test_extra_hierarchy(self): # note the conflicting "hello3" key in higher level: innerdict = {"hello3": "hello4"} adict = {"hello1": "hello2", "hello5": innerdict, "hello3": "hello6"} cei_events.event("unittest", "TRIAL1", self.log, extra=adict) events = cei_events.events_from_file(self.logfilepath) event = events[0] assert event.extra["hello1"] == "hello2" assert event.extra["hello3"] == "hello6" innerdict = event.extra["hello5"] assert isinstance(innerdict, dict) assert innerdict["hello3"] == "hello4"
def op_work(self, content, headers, msg): if not self.laterinitialized: log.error("message got here without the later-init") sleepsecs = int(content['work_amount']) extradict = {"batchid":content['batchid'], "jobid":content['jobid'], "work_amount":sleepsecs} cei_events.event("worker", "job_begin", extra=extradict) log.info("WORK: sleeping for %d seconds ---" % sleepsecs) yield pu.asleep(sleepsecs) yield self.reply(msg, 'result', {'result':'work_complete'}, {}) cei_events.event("worker", "job_end", extra=extradict)
def test_timestamp(self): utc_now = datetime.datetime.utcnow() cei_events.event("unittest", "TRIAL1", self.log) events = cei_events.events_from_file(self.logfilepath) assert len(events) == 1 ts = events[0].timestamp # It is possible that any of these values could have rolled over # between acquiring utc_now and recording the event. But this is # unlikely enough that we'll keep this important UTC sanity check: assert ts.year == utc_now.year assert ts.month == utc_now.month assert ts.day == utc_now.day assert ts.hour == utc_now.hour
def test_extra_hierarchy(self): # note the conflicting "hello3" key in higher level: innerdict = {"hello3": "hello4"} adict = {"hello1": "hello2", "hello5": innerdict, "hello3": "hello6"} cei_events.event("unittest", "TRIAL1", log, extra=adict) events = cei_events.events_from_file(self.logfilepath) assert len(events) == 1 assert events[0].extra["hello1"] == "hello2" assert events[0].extra["hello3"] == "hello6" innerdict = events[0].extra["hello5"] assert isinstance(innerdict, dict) assert innerdict["hello3"] == "hello4"
def test_timestamp(self): utc_now = datetime.datetime.utcnow() cei_events.event("unittest", "TRIAL1", log) events = cei_events.events_from_file(self.logfilepath) assert len(events) == 1 ts = events[0].timestamp # It is possible that any of these values could have rolled over # between acquiring utc_now and recording the event. But this is # unlikely enough that we'll keep this important UTC sanity check: assert ts.year == utc_now.year assert ts.month == utc_now.month assert ts.day == utc_now.day assert ts.hour == utc_now.hour
def op_work(self, content, headers, msg): if not self.laterinitialized: log.error("message got here without the later-init") sleepsecs = int(content['work_amount']) extradict = { "batchid": content['batchid'], "jobid": content['jobid'], "work_amount": sleepsecs } cei_events.event("worker", "job_begin", extra=extradict) log.info("WORK: sleeping for %d seconds ---" % sleepsecs) yield pu.asleep(sleepsecs) yield self.reply(msg, 'result', {'result': 'work_complete'}, {}) cei_events.event("worker", "job_end", extra=extradict)
def add_domain(self, caller, domain_id, definition_id, config, subscriber_name=None, subscriber_op=None): """See: EPUManagement.msg_add_domain() """ # TODO: parameters are from messages, do legality checks here # assert that engine_conf['epuworker_type']['sleeper'] is owned by caller log.debug("ADD Domain: %s", config) # Make a copy of the definition and merge the config inside. # If a definition is changed or deleted, it has no impact over any of # the domains. try: definition = self.store.get_domain_definition(definition_id) except NotFoundError: raise ValueError("Domain definition does not exist: %s" % definition_id) merged_config = copy.copy(definition.get_definition()) # Hand-made deep merge of config into definition if EPUM_CONF_GENERAL in config: if EPUM_CONF_GENERAL in merged_config: merged_config[EPUM_CONF_GENERAL].update(config[EPUM_CONF_GENERAL]) else: merged_config[EPUM_CONF_GENERAL] = config[EPUM_CONF_GENERAL] if EPUM_CONF_HEALTH in config: if EPUM_CONF_HEALTH in merged_config: merged_config[EPUM_CONF_HEALTH].update(config[EPUM_CONF_HEALTH]) else: merged_config[EPUM_CONF_HEALTH] = config[EPUM_CONF_HEALTH] if EPUM_CONF_ENGINE in config: if EPUM_CONF_ENGINE in merged_config: merged_config[EPUM_CONF_ENGINE].update(config[EPUM_CONF_ENGINE]) else: merged_config[EPUM_CONF_ENGINE] = config[EPUM_CONF_ENGINE] self._validate_engine_config(merged_config) with EpuLoggerThreadSpecific(domain=domain_id, user=caller): domain = self.store.add_domain(caller, domain_id, merged_config) if subscriber_name and subscriber_op: domain.add_subscriber(subscriber_name, subscriber_op) extradict = {'user': caller, 'domain_id': domain_id, 'definition_id': definition_id} cei_events.event("epumanagement", "new_domain", extra=extradict)
class ProvisionerService(ServiceProcess): """Provisioner service interface """ # Declaration of service declare = ServiceProcess.service_declare(name='provisioner', version='0.1.0', dependencies=[]) @defer.inlineCallbacks def slc_init(self): cei_events.event("provisioner", "init_begin") try: store = self.spawn_args['store'] site_drivers = self.spawn_args['site_drivers'] context_client = self.spawn_args['context_client'] except KeyError, e: raise KeyError("Missing provisioner spawn_arg: " + str(e)) self.store = store notifier = self.spawn_args.get('notifier') self.notifier = notifier or ProvisionerNotifier(self) self.dtrs = DeployableTypeRegistryClient(self) self.core = ProvisionerCore(self.store, self.notifier, self.dtrs, site_drivers, context_client) yield self.core.recover() cei_events.event("provisioner", "init_end") # operator can disable new launches self.enabled = True self.terminate_all_deferred = None
def test_manual_event_write(self): cruft = "some cruft %s" % cei_events.event_logtxt("unittest", "TRIAL1") log.warning(cruft) events = cei_events.events_from_file(self.logfilepath) assert len(events) == 1 cei_events.event("unittest", "TRIAL2", log) events = cei_events.events_from_file(self.logfilepath) assert len(events) == 2 cruft = "cruft2 %s" % cei_events.event_logtxt("unittest", "TRIAL3") log.warning(cruft) events = cei_events.events_from_file(self.logfilepath) assert len(events) == 3 found = {"TRIAL1": False, "TRIAL2": False, "TRIAL3": False} for ev in events: if found.has_key(ev.name): found[ev.name] = True for val in found.values(): assert val
def __init__(self, *args, **kwargs): configs = ["service", "epu_worker"] config_files = get_config_paths(configs) self.CFG = bootstrap.configure(config_files) self.log = logging.getLogger() try: if os.environ.get('EPU_USE_GEVENT'): bootstrap.enable_gevent() else: self.log.info("Using standard python Threading") except: self.log.warning("gevent not available. Falling back to threading") self.queue_name_work = self.CFG.queue_name_work extradict = {"queue_name_work": self.queue_name_work} cei_events.event("worker", "init_begin", extra=extradict) self.dashi = bootstrap.dashi_connect(self.topic, self.CFG)
def test_manual_event_write(self): cruft = "some cruft %s" % cei_events.event_logtxt("unittest", "TRIAL1") self.log.warning(cruft) events = cei_events.events_from_file(self.logfilepath) assert len(events) == 1 cei_events.event("unittest", "TRIAL2", self.log) events = cei_events.events_from_file(self.logfilepath) assert len(events) == 2 cruft = "cruft2 %s" % cei_events.event_logtxt("unittest", "TRIAL3") self.log.warning(cruft) events = cei_events.events_from_file(self.logfilepath) assert len(events) == 3 found = {"TRIAL1": False, "TRIAL2": False, "TRIAL3": False} for ev in events: if ev.name in found: found[ev.name] = True for val in found.values(): assert val
def launch(self, deployable_type_id, site, allocation, count=1, extravars=None): """ Choose instance IDs for each instance desired, a launch ID and send appropriate message to Provisioner. @param deployable_type_id string identifier of the DT to launch @param site IaaS site to launch on @param allocation IaaS allocation (size) to request @param count number of instances to launch @param extravars Optional, see engine implementer's guide @retval tuple (launch_id, instance_ids), see guide @exception Exception illegal input @exception Exception message not sent """ # right now we are sending some node-specific data in provisioner vars # (node_id at least) if count != 1: raise NotImplementedError("Only single-node launches are supported") launch_id = str(uuid.uuid4()) log.info("Request for DT '%s' is a new launch with id '%s'", deployable_type_id, launch_id) if extravars: extravars = deepcopy(extravars) else: extravars = None new_instance_id = str(uuid.uuid4()) instance = self.domain.new_instance_launch(deployable_type_id, new_instance_id, launch_id, site, allocation, extravars=extravars) new_instance_id_list = (new_instance_id,) self.execute_instance_launch(instance) extradict = {"launch_id": launch_id, "new_instance_ids": new_instance_id_list} cei_events.event("controller", "new_launch", extra=extradict) return launch_id, new_instance_id_list
def test_unique_keys(self): cei_events.event("unittest", "NAME", self.log) cei_events.event("unittest", "NAME", self.log) cei_events.event("unittest", "NAME", self.log) cei_events.event("unittest", "NAME", self.log) cei_events.event("unittest", "NAME", self.log) cei_events.event("unittest", "NAME", self.log) cei_events.event("unittest", "NAME", self.log) events = cei_events.events_from_file(self.logfilepath) assert len(events) == 7 uniqs = {} for ev in events: uniqs[ev.timestamp] = None assert len(uniqs) == 7
def later_init(self): spawnId = yield self.workReceiver.attach() log.debug("spawnId: %s" % spawnId) self.laterinitialized = True extradict = {"queue_name_work":self.queue_name_work} cei_events.event("worker", "init_end", extra=extradict)
def slc_init(self): scoped_name = self.get_scoped_name("system", self.svc_name) self.scoped_name = scoped_name queue_name_work = self.spawn_args.get("queue_name_work") if queue_name_work: self.queue_name_work = self.get_scoped_name( "system", queue_name_work) extradict = {"queue_name_work": self.queue_name_work} cei_events.event(self.svc_name, "init_begin", extra=extradict) yield self._make_queue(queue_name_work) queuestat_client = QueueStatClient(self) yield queuestat_client.watch_queue(self.queue_name_work, self.scoped_name, 'sensor_info') cei_events.event(self.svc_name, "queue_watched") else: self.worker_queue_receiver = None self.queue_name_work = None extradict = None cei_events.event(self.svc_name, "init_begin", extra=extradict) engineclass = "epu.decisionengine.impls.NpreservingEngine" if self.spawn_args.has_key("engine_class"): engineclass = self.spawn_args["engine_class"] log.info("Using configured decision engine: %s" % engineclass) else: log.info("Using default decision engine: %s" % engineclass) if self.spawn_args.has_key("engine_conf"): engine_conf = self.spawn_args["engine_conf"] if isinstance(engine_conf, str): engine_conf = json.loads(engine_conf) else: engine_conf = None if self.spawn_args.has_key("cassandra"): cass = self.spawn_args["cassandra"] host = cass['hostname'] username = cass['username'] password = cass['password'] port = cass['port'] keyspace = cass['keyspace'] store = CassandraControllerStore(self.svc_name, host, port, username, password, keyspace, CoreInstance, SensorItem) store.initialize() store.activate() elif self.spawn_args.has_key('store'): store = self.spawn_args['store'] else: store = ControllerStore() self.core = ControllerCore(ProvisionerClient(self), engineclass, scoped_name, conf=engine_conf, store=store) # run state recovery and engine initialization # this one needs to run before any messages start arriving. It pulls # information from persistence and refreshes local caches. yield self.core.run_recovery() # temporarily doing this later due to a potential bug in ioncore where # queues may not be bound before slc_init runs. This means if the # provisioner is quck to reply to dump_state some messages may be # missed. reactor.callLater(1, self._delayed_init)
def launch(self, deployable_type_id, launch_description, extravars=None): """Choose instance IDs for each instance desired, a launch ID and send appropriate message to Provisioner. Control API method, see the decision engine implementer's guide. @param deployable_type_id string identifier of the DP to launch @param launch_description See engine implementer's guide @param extravars Optional, see engine implementer's guide @retval tuple (launch_id, launch_description), see guide @exception Exception illegal input @exception Exception message not sent """ # right now we are sending some node-specific data in provisioner vars # (node_id at least) if len(launch_description) != 1: raise NotImplementedError( "Only single-node launches are supported") launch_id = str(uuid.uuid4()) log.info("Request for DP '%s' is a new launch with id '%s'" % (deployable_type_id, launch_id)) new_instance_id_list = [] for group, item in launch_description.iteritems(): log.info( " - %s is %d %s from %s" % (group, item.num_instances, item.allocation_id, item.site)) if item.num_instances != 1: raise NotImplementedError( "Only single-node launches are supported") for i in range(item.num_instances): new_instance_id = str(uuid.uuid4()) self.state.new_instance_launch(new_instance_id, launch_id, item.site, item.allocation_id) item.instance_ids.append(new_instance_id) new_instance_id_list.append(new_instance_id) vars_send = self.prov_vars.copy() if extravars: vars_send.update(extravars) # The node_id var is the reason only single-node launches are supported. # It could be instead added by the provisioner or something? It also # is complicated by the contextualization system. vars_send['node_id'] = new_instance_id_list[0] vars_send['heartbeat_dest'] = self.controller_name # hide passwords from logging hide_password = deepcopy(vars_send) if 'cassandra_password' in hide_password: hide_password['cassandra_password'] = '******' if 'broker_password' in hide_password: hide_password['broker_password'] = '******' log.debug("Launching with parameters:\n%s" % str(hide_password)) subscribers = (self.controller_name, ) self.provisioner.provision(launch_id, deployable_type_id, launch_description, subscribers, vars=vars_send) extradict = { "launch_id": launch_id, "new_instance_ids": new_instance_id_list, "subscribers": subscribers } cei_events.event("controller", "new_launch", extra=extradict) return launch_id, launch_description
def test_event_sourcefilter2(self): cei_events.event("SRC1", "NM1", self.log) self.log.debug("something not an event") cei_events.event("SRX2", "NM2", self.log) cei_events.event("SRC3", "NM3", self.log) self.log.debug("something not an event") cei_events.event("SRX4", "NM4", self.log) cei_events.event("SRC5", "NM5", self.log) cei_events.event("SRC6", "NM6", self.log) path = self.logfilepath events = cei_events.events_from_file(path, sourcefilter="SRC") assert len(events) == 4
def test_event_namefilter2(self): cei_events.event("unittest", "NM1", log) log.debug("something not an event") cei_events.event("unittest", "XX2", log) cei_events.event("unittest", "NM3", log) log.debug("something not an event") cei_events.event("unittest", "XX4", log) cei_events.event("unittest", "NM5", log) cei_events.event("unittest", "XX6", log) path = self.logfilepath events = cei_events.events_from_file(path, namefilter="NM") assert len(events) == 3
def op_de_state(self, content, headers, msg): state = self.core.de_state() extradict = {"state": state} cei_events.event(self.svc_name, "de_state", extra=extradict) yield self.reply_ok(msg, state)
def test_event_nameandsourcefilter(self): cei_events.event("SRC1", "NX1", log) log.debug("something not an event") cei_events.event("SRX2", "NM2", log) cei_events.event("SRC3", "XX3", log) cei_events.event("SRX4", "XX4", log) cei_events.event("SRC5", "NM5", log) log.debug("something not an event") cei_events.event("SRC6", "NM6", log) path = self.logfilepath events = cei_events.events_from_file(path, sourcefilter="SRC", namefilter="NM") assert len(events) == 2
def _delayed_init(self): yield self.core.run_initialize() self.core.begin_controlling() cei_events.event(self.svc_name, "init_end")
def test_extra(self): adict = {"hello1":"hello2"} cei_events.event("unittest", "TRIAL1", log, extra=adict) events = cei_events.events_from_file(self.logfilepath) assert len(events) == 1 assert events[0].extra["hello1"] == "hello2"
def test_unique_keys(self): cei_events.event("unittest", "NAME", log) cei_events.event("unittest", "NAME", log) cei_events.event("unittest", "NAME", log) cei_events.event("unittest", "NAME", log) cei_events.event("unittest", "NAME", log) cei_events.event("unittest", "NAME", log) cei_events.event("unittest", "NAME", log) events = cei_events.events_from_file(self.logfilepath) assert len(events) == 7 uniqs = {} for ev in events: uniqs[ev.timestamp] = None assert len(uniqs) == 7
def test_event_nameandsourcefilter(self): cei_events.event("SRC1", "NX1", self.log) self.log.debug("something not an event") cei_events.event("SRX2", "NM2", self.log) cei_events.event("SRC3", "XX3", self.log) cei_events.event("SRX4", "XX4", self.log) cei_events.event("SRC5", "NM5", self.log) self.log.debug("something not an event") cei_events.event("SRC6", "NM6", self.log) path = self.logfilepath events = cei_events.events_from_file(path, sourcefilter="SRC", namefilter="NM") assert len(events) == 2
def test_extra_integer_values(self): adict = {"hello1": 34} cei_events.event("unittest", "TRIAL1", log, extra=adict) events = cei_events.events_from_file(self.logfilepath) assert len(events) == 1 assert events[0].extra["hello1"] == 34
def op_de_state(self, content, headers, msg): state = self.core.de_state() extradict = {"state":state} cei_events.event(self.svc_name, "de_state", extra=extradict) yield self.reply_ok(msg, state)
def later_init(self): spawnId = yield self.workReceiver.attach() log.debug("spawnId: %s" % spawnId) self.laterinitialized = True extradict = {"queue_name_work": self.queue_name_work} cei_events.event("worker", "init_end", extra=extradict)
def slc_init(self): scoped_name = self.get_scoped_name("system", self.svc_name) self.scoped_name = scoped_name queue_name_work = self.spawn_args.get("queue_name_work") if queue_name_work: self.queue_name_work = self.get_scoped_name("system", queue_name_work) extradict = {"queue_name_work":self.queue_name_work} cei_events.event(self.svc_name, "init_begin", extra=extradict) yield self._make_queue(queue_name_work) queuestat_client = QueueStatClient(self) yield queuestat_client.watch_queue(self.queue_name_work, self.scoped_name, 'sensor_info') cei_events.event(self.svc_name, "queue_watched") else: self.worker_queue_receiver = None self.queue_name_work = None extradict = None cei_events.event(self.svc_name, "init_begin", extra=extradict) engineclass = "epu.decisionengine.impls.NpreservingEngine" if self.spawn_args.has_key("engine_class"): engineclass = self.spawn_args["engine_class"] log.info("Using configured decision engine: %s" % engineclass) else: log.info("Using default decision engine: %s" % engineclass) if self.spawn_args.has_key("engine_conf"): engine_conf = self.spawn_args["engine_conf"] if isinstance(engine_conf, str): engine_conf = json.loads(engine_conf) else: engine_conf = None if self.spawn_args.has_key("cassandra"): cass = self.spawn_args["cassandra"] host = cass['hostname'] username = cass['username'] password = cass['password'] port = cass['port'] keyspace = cass['keyspace'] store = CassandraControllerStore(self.svc_name, host, port, username, password, keyspace, CoreInstance, SensorItem) store.initialize() store.activate() elif self.spawn_args.has_key('store'): store = self.spawn_args['store'] else: store = ControllerStore() self.core = ControllerCore(ProvisionerClient(self), engineclass, scoped_name, conf=engine_conf, store=store) # run state recovery and engine initialization # this one needs to run before any messages start arriving. It pulls # information from persistence and refreshes local caches. yield self.core.run_recovery() # temporarily doing this later due to a potential bug in ioncore where # queues may not be bound before slc_init runs. This means if the # provisioner is quck to reply to dump_state some messages may be # missed. reactor.callLater(1, self._delayed_init)
message = '%s nodes from IaaS launch but %s were expected' % ( len(iaas_nodes), len(nodes)) log.error(message) raise ProvisioningError('IAAS_PROBLEM '+ message) for node_rec, iaas_node in izip(nodes, iaas_nodes): node_rec['iaas_id'] = iaas_node.id update_node_ip_info(node_rec, iaas_node) node_rec['state'] = states.PENDING node_rec['pending_timestamp'] = time.time() extradict = {'public_ip': node_rec.get('public_ip'), 'iaas_id': iaas_node.id, 'node_id': node_rec['node_id']} cei_events.event("provisioner", "new_node", extra=extradict) @defer.inlineCallbacks def store_and_notify(self, records, subscribers): """Convenience method to store records and notify subscribers. """ yield self.store.put_nodes(records) yield self.notifier.send_records(records, subscribers) @defer.inlineCallbacks def dump_state(self, nodes, force_subscribe=None): """Resends node state information to subscribers @param nodes list of node IDs @param force_subscribe optional, an extra subscriber that may not be listed in local node records """
def test_extra_integer_values(self): adict = {"hello1": 34} cei_events.event("unittest", "TRIAL1", self.log, extra=adict) events = cei_events.events_from_file(self.logfilepath) assert len(events) == 1 assert events[0].extra["hello1"] == 34
def test_event_namefilter2(self): cei_events.event("unittest", "NM1", self.log) self.log.debug("something not an event") cei_events.event("unittest", "XX2", self.log) cei_events.event("unittest", "NM3", self.log) self.log.debug("something not an event") cei_events.event("unittest", "XX4", self.log) cei_events.event("unittest", "NM5", self.log) cei_events.event("unittest", "XX6", self.log) path = self.logfilepath events = cei_events.events_from_file(path, namefilter="NM") assert len(events) == 3
def test_event_sourcefilter2(self): cei_events.event("SRC1", "NM1", log) log.debug("something not an event") cei_events.event("SRX2", "NM2", log) cei_events.event("SRC3", "NM3", log) log.debug("something not an event") cei_events.event("SRX4", "NM4", log) cei_events.event("SRC5", "NM5", log) cei_events.event("SRC6", "NM6", log) path = self.logfilepath events = cei_events.events_from_file(path, sourcefilter="SRC") assert len(events) == 4