def _publish_event(self, event_msg, origin, event_type=None): event_type = event_type or self.event_type or event_msg._get_type() assert origin and event_type topic = self._topic(event_type, origin, base_types=event_msg.base_types, sub_type=event_msg.sub_type, origin_type=event_msg.origin_type) to_name = (self._send_name.exchange, topic) log.debug("Publishing event message to %s", to_name) try: ep = self.publish(event_msg, to_name=to_name) ep.close() except Exception as ex: log.exception("Failed to publish event '%s'" % (event_msg)) return False try: # store published event but only if we specified an event_repo if self.event_repo: self.event_repo.put_event(event_msg) except Exception as ex: log.exception("Failed to store published event '%s'" % (event_msg)) return False return True
def start_app(self, appdef=None, config=None): """ @brief Start an app from an app definition. Note: apps can come in one of 2 variants: 1 processapp: In-line defined process to be started 2 regular app: Full app definition """ log.debug("AppManager.start_app(appdef=%s) ..." % appdef) appdef = DotDict(appdef) app_config = DictModifier(CFG) if 'config' in appdef: # Apply config from app file app_file_cfg = DotDict(appdef.config) app_config.update(app_file_cfg) if config: # Nest dict modifier and apply config from rel file app_config = DictModifier(app_config, config) if 'processapp' in appdef: # Case 1: Appdef contains definition of process to start name, module, cls = appdef.processapp try: pid = self.container.spawn_process(name, module, cls, app_config) appdef._pid = pid self.apps.append(appdef) except Exception, ex: log.exception("Appl %s start from processapp failed" % appdef.name)
def get_valid_principal_commitments(principal_id=None, consumer_id=None): """ Returns the list of valid commitments for the specified principal (org or actor. If optional consumer_id (actor) is supplied, then filtered by consumer_id """ log.debug("Finding commitments for principal: %s", principal_id) if principal_id is None: return None try: gov_controller = bootstrap.container_instance.governance_controller commitments, _ = gov_controller.rr.find_objects(principal_id, PRED.hasCommitment, RT.Commitment, id_only=False) if not commitments: return None cur_time = get_ion_ts_millis() commitment_list = [com for com in commitments if (consumer_id == None or com.consumer == consumer_id) and \ (int(com.expiration) == 0 or (int(com.expiration) > 0 and cur_time < int(com.expiration)))] if commitment_list: return commitment_list except Exception: log.exception("Could not determine actor resource commitments") return None
def get_valid_resource_commitments(resource_id=None, actor_id=None): """ Returns the list of valid commitments for the specified resource. If optional actor_id is supplied, then filtered by actor_id """ log.debug("Finding commitments for resource_id: %s and actor_id: %s", resource_id, actor_id) if resource_id is None: return None try: gov_controller = bootstrap.container_instance.governance_controller commitments, _ = gov_controller.rr.find_subjects(RT.Commitment, PRED.hasTarget, resource_id, id_only=False) if not commitments: return None cur_time = get_ion_ts_millis() commitment_list = [com for com in commitments if (actor_id == None or com.consumer == actor_id) and \ (int(com.expiration) == 0 or (int(com.expiration) > 0 and cur_time < int(com.expiration)))] if commitment_list: return commitment_list except Exception: log.exception("Could not determine actor resource commitments") return None
def listen(self, binding=None): log.debug("LEF.listen: binding %s", binding) binding = binding or self._binding or self.name[1] self._ensure_node() self._chan = self.node.channel(self.channel_type) self._setup_listener(self.name, binding=binding) self._chan.start_consume() # notify any listeners of our readiness self._ready_event.set() while True: log.debug("LEF: %s blocking, waiting for a message" % str(self.name)) try: newchan = self._chan.accept() msg, headers, delivery_tag = newchan.recv() log.debug("LEF %s received message %s, headers %s, delivery_tag %s", self.name, msg, headers, delivery_tag) except ChannelClosedError as ex: log.debug('Channel was closed during LEF.listen') break try: e = self.create_endpoint(existing_channel=newchan) e._message_received(msg, headers) except Exception: log.exception("Unhandled error while handling received message") raise finally: # ALWAYS ACK newchan.ack(delivery_tag)
def build_service_map(self): """ Adds all known service definitions to service registry. @todo: May be a bit fragile due to using BaseService.__subclasses__ """ for cls in BaseService.__subclasses__(): assert hasattr(cls, 'name'), 'Service class must define name value. Service class in error: %s' % cls if cls.name: self.services_by_name[cls.name] = cls self.add_servicedef_entry(cls.name, "base", cls) try: self.add_servicedef_entry(cls.name, "schema", json.loads(cls.SCHEMA_JSON)) except Exception as ex: log.exception("Cannot parse service schema " + cls.name) interfaces = list(implementedBy(cls)) if interfaces: self.add_servicedef_entry(cls.name, "interface", interfaces[0]) if cls.__name__.startswith("Base"): try: client = "%s.%sProcessClient" % (cls.__module__, cls.__name__[4:]) self.add_servicedef_entry(cls.name, "client", named_any(client)) sclient = "%s.%sClient" % (cls.__module__, cls.__name__[4:]) self.add_servicedef_entry(cls.name, "simple_client", named_any(sclient)) except Exception, ex: log.warning("Cannot find client for service %s" % (cls.name))
def _call_callbacks(self, group, action, cb_data): cbs = self._stats_callbacks[group] for cb in cbs: try: cb(group, action, cb_data) except Exception: log.exception("Error in stats callback")
def serve_forever(self): """ Run the container until killed. """ log.debug("In Container.serve_forever") if not self.proc_manager.proc_sup.running: self.start() # serve forever short-circuits if immediate is on and children len is ok num_procs = len(self.proc_manager.proc_sup.children) immediate = CFG.system.get('immediate', False) if not (immediate and num_procs == 1): # only spawned greenlet is the CC-Agent # print a warning just in case if immediate and num_procs != 1: log.warn("CFG.system.immediate=True but number of spawned processes is not 1 (%d)", num_procs) try: # This just waits in this Greenlet for all child processes to complete, # which is triggered somewhere else. self.proc_manager.proc_sup.join_children() except (KeyboardInterrupt, SystemExit) as ex: log.info('Received a kill signal, shutting down the container.') if hasattr(self, 'gl_parent_watch') and self.gl_parent_watch is not None: self.gl_parent_watch.kill() except: log.exception('Unhandled error! Forcing container shutdown') else: log.debug("Container.serve_forever short-circuiting due to CFG.system.immediate") self.proc_manager.proc_sup.shutdown(CFG.cc.timeout.shutdown)
def execute_retrieve(self): """ execute_retrieve Executes a retrieval and returns the result as a value in lieu of publishing it on a stream """ try: coverage = DatasetManagementService._get_coverage(self.dataset_id, mode="r") if coverage.num_timesteps == 0: log.info("Reading from an empty coverage") rdt = RecordDictionaryTool(param_dictionary=coverage.parameter_dictionary) else: rdt = self._coverage_to_granule( coverage=coverage, start_time=self.start_time, end_time=self.end_time, stride_time=self.stride_time, parameters=self.parameters, tdoa=self.tdoa, ) except: log.exception("Problems reading from the coverage") raise BadRequest("Problems reading from the coverage") finally: coverage.close(timeout=5) return rdt.to_granule()
def outgoing(self, invocation): #log.trace("PolicyInterceptor.outgoing: %s", invocation.get_arg_value('process', invocation)) #Check for a field with the ResourceId decorator and if found, then set resource-id # in the header with that field's value or if the decorator specifies a field within an object, #then use the object's field value ( ie. _id) try: if isinstance(invocation.message, IonObjectBase): decorator = 'ResourceId' field = invocation.message.find_field_for_decorator(decorator) if field is not None and hasattr(invocation.message,field): deco_value = invocation.message.get_decorator_value(field, decorator) if deco_value: #Assume that if there is a value, then it is specifying a field in the object fld_value = getattr(invocation.message,field) if getattr(fld_value, deco_value) is not None: invocation.headers['resource-id'] = getattr(fld_value, deco_value) else: if getattr(invocation.message,field) is not None: invocation.headers['resource-id'] = getattr(invocation.message,field) except Exception, ex: log.exception(ex)
def serve_forever(self): """ Run the container until killed. """ log.debug("In Container.serve_forever") if not self.proc_manager.proc_sup.running: self.start() # Exit if immediate==True and children len is ok num_procs = len(self.proc_manager.proc_sup.children) immediate = CFG.system.get('immediate', False) if immediate and num_procs == 1: # only spawned greenlet is the CC-Agent log.debug("Container.serve_forever exiting due to CFG.system.immediate") else: # print a warning just in case if immediate and num_procs != 1: log.warn("CFG.system.immediate=True but number of spawned processes is not 1 (%d)", num_procs) try: # This just waits in this Greenlet for all child processes to complete, # which is triggered somewhere else. self.proc_manager.proc_sup.join_children() except (KeyboardInterrupt, SystemExit) as ex: if hasattr(self, 'gl_parent_watch') and self.gl_parent_watch is not None: # Remove the greenlet that watches the parent process self.gl_parent_watch.kill() # Let the caller handle this raise except: log.exception('Unhandled error! Forcing container shutdown')
def _run_consumer(self, ctag, queue_name, gqueue, callback): cnt = 0 while True: m = gqueue.get() if isinstance(m, self.ConsumerClosedMessage): break exchange, routing_key, body, props = m # create method frame method_frame = DotDict() method_frame['consumer_tag'] = ctag method_frame['redelivered'] = False # @TODO method_frame['exchange'] = exchange method_frame['routing_key'] = routing_key # create header frame header_frame = DotDict() header_frame['headers'] = props.copy() # make delivery tag for ack/reject later dtag = self._generate_dtag(ctag, cnt) cnt += 1 with self._lock_unacked: self._unacked[dtag] = (ctag, queue_name, m) method_frame['delivery_tag'] = dtag # deliver to callback try: callback(self, method_frame, header_frame, body) except Exception: log.exception("delivering to consumer, ignore!")
def outgoing(self, invocation): payload = invocation.message # Compliance: Make sure sent message objects support DotDict as arguments. # Although DotDict is subclass of dict, msgpack does not like it if isinstance(payload, IonMessageObjectBase): for k, v in payload.__dict__.iteritems(): if isinstance(v, DotDict): setattr(payload, k, v.as_dict()) # Msgpack the content to binary str - does nested IonObject encoding try: invocation.message = msgpack.packb(payload, default=encode_ion) except Exception: log.exception("Illegal type in IonObject attributes: %s", payload) raise BadRequest("Illegal type in IonObject attributes") # Make sure no Nones exist in headers - this indicates a problem somewhere up the stack. # pika will choke hard on them as well, masking the actual problem, so we catch here. nonelist = [(k, v) for k, v in invocation.headers.iteritems() if v is None] if nonelist: raise BadRequest("Invalid headers containing None values: %s" % str(nonelist)) msg_size = len(invocation.message) if msg_size > self.max_message_size: raise BadRequest('The message size %s is larger than the max_message_size value of %s' % ( msg_size, self.max_message_size)) return invocation
def handle_dds(self, coverage, dataset, fields): cov = coverage try: time_name = coverage.temporal_parameter_name time_context = coverage.get_parameter_context(time_name) time_attrs = self.get_attrs(cov, time_name) time_base = BaseType(time_name, type=self.dap_type(time_context), attributes=time_attrs, dimensions=(time_name,), shape=(coverage.num_timesteps,)) dataset[time_name] = time_base except: log.exception('Problem reading cov %s', str(cov)) raise # Can't do much without time for var in fields: while var: name, slice_ = var.pop(0) name = urllib.unquote(name) if name == time_name: continue # Already added to the dataset try: grid = GridType(name=name) context = coverage.get_parameter_context(name) attrs = self.get_attrs(cov, name) grid[name] = BaseType(name=name, type=self.dap_type(context), attributes=attrs, dimensions=(time_name,), shape=(coverage.num_timesteps,)) grid[cov.temporal_parameter_name] = time_base dataset[name] = grid except Exception: log.exception('Problem reading cov %s', str(cov)) continue return dataset
def start_app(self, appdef=None, config=None): """ @brief Start an app from an app definition. Note: apps can come in one of 2 variants: 1 processapp: In-line defined process to be started 2 regular app: Full app definition """ log.debug("AppManager.start_app(appdef=%s) ..." % appdef) appdef = DotDict(appdef) if 'config' in appdef: app_cfg = appdef.config.copy() if config: dict_merge(app_cfg, config, inplace=True) config = app_cfg if 'processapp' in appdef: # Case 1: Appdef contains definition of process to start name, module, cls = appdef.processapp try: pid = self.container.spawn_process(name, module, cls, config) appdef._pid = pid self.apps.append(appdef) except Exception, ex: log.exception("Appl %s start from processapp failed" % appdef.name)
def get_version_info(): import pkg_resources pkg_list = ["coi-services", "pyon", "coverage-model", "ion-functions", "eeagent", "epu", "utilities", "marine-integrations"] version = {} for package in pkg_list: try: version["%s-release" % package] = pkg_resources.require(package)[0].version # @TODO git versions for each? except pkg_resources.DistributionNotFound: pass try: dir_client = DirectoryServiceProcessClient(process=service_gateway_instance) sys_attrs = dir_client.lookup("/System") if sys_attrs and isinstance(sys_attrs, dict): version.update({k: v for (k, v) in sys_attrs.iteritems() if "version" in k.lower()}) except Exception as ex: log.exception("Could not determine system directory attributes") return gateway_json_response(version)
def publish_heartbeat(self): try: hb_msg = self.get_heartbeat_message() headers = dict(expiration=60000) self.heartbeat_pub.publish(hb_msg, headers=headers) except Exception: log.exception("Error publishing heatbeat")
def stop(self, do_exit=True): log.info("=============== Container stopping... ===============") self._status = TERMINATING if self.has_capability(CCAP.EVENT_PUBLISHER) and self.event_pub is not None: try: self.event_pub.publish_event(event_type="ContainerLifecycleEvent", origin=self.id, origin_type="CapabilityContainer", sub_type="TERMINATE", state=ContainerStateEnum.TERMINATE) except Exception as ex: log.exception("Error sending event") while self._capabilities: capability = self._capabilities.pop() #log.debug("stop(): Stopping '%s'" % capability) try: cap_obj = self._cap_instances[capability] cap_obj.stop() del self._cap_instances[capability] except Exception as ex: log.exception("Container stop(): Error stop %s" % capability) Container.instance = None from pyon.core import bootstrap bootstrap.container_instance = None self._is_started = False self._status = TERMINATED log.info("Container stopped (%s).", self.id) if do_exit: os.kill(os.getpid(), signal.SIGTERM)
def _read_interval_time(self): """ Reads the hsflowd conf file to determine what time should be used. """ if not (self._hsflowd_addr == "localhost" or self._hsflowd_addr == "127.0.0.1"): log.debug("Skipping reading hsflow auto file, hsflowd is not running locally") else: try: mtime = os.stat(self._hsflowd_conf).st_mtime except OSError: # if you can't stat it, you can't read it most likely log.info("Could not stat hsflowd.auto file") return if mtime != self._conf_last_mod: self._conf_last_mod = mtime # appears to be simple key=value, one per line try: with open(self._hsflowd_conf) as f: while True: c = f.readline() if c == "": break elif c.startswith("polling="): self._counter_interval = int(c.rstrip().split("=")[1]) log.debug("New polling interval time: %d", self._counter_interval) break except IOError: log.exception("Could not open/read hsflowd.auto")
def stop(self): log.info("=============== Container stopping... ===============") if self.event_pub is not None: try: self.event_pub.publish_event(event_type="ContainerLifecycleEvent", origin=self.id, origin_type="CapabilityContainer", sub_type="TERMINATE", state=ContainerStateEnum.TERMINATE) except Exception as ex: log.exception(ex) while self._capabilities: capability = self._capabilities.pop() log.debug("stop(): Stopping '%s'" % capability) try: self._stop_capability(capability) except Exception as ex: log.exception("Container stop(): Error stop %s" % capability) Container.instance = None from pyon.core import bootstrap bootstrap.container_instance = None self._is_started = False log.debug("Container stopped, OK.")
def message_received(self, msg, headers): assert self._routing_obj, "How did I get created without a routing object?" log.debug("RPCResponseEndpointUnit.message_received\n\tmsg: %s\n\theaders: %s", msg, headers) cmd_arg_obj = msg cmd_op = headers.get('op', None) # transform cmd_arg_obj into a dict if hasattr(cmd_arg_obj, '__dict__'): cmd_arg_obj = cmd_arg_obj.__dict__ elif isinstance(cmd_arg_obj, dict): pass else: raise BadRequest("Unknown message type, cannot convert into kwarg dict: %s" % str(type(cmd_arg_obj))) # op name must exist! if not hasattr(self._routing_obj, cmd_op): raise BadRequest("Unknown op name: %s" % cmd_op) ro_meth = getattr(self._routing_obj, cmd_op) result = None response_headers = {} try: result = ro_meth(**cmd_arg_obj) response_headers = { 'status_code': 200, 'error_message': '' } except TypeError as ex: log.exception("TypeError while attempting to call routing object's method") response_headers = self._create_error_response(ServerError(ex.message)) return result, response_headers
def stop_app(self, appdef): log.debug("App '%s' stopping" % appdef.name) try: if '_mod_loaded' in appdef: appdef._mod_loaded.stop(self.container, appdef._state) except Exception, ex: log.exception("Application %s stop failed" % appdef.name)
def _unbind(self, exchange_point, exchange_name, binding_key): xp = self.container.ex_manager.create_xp(exchange_point) xn = self.container.ex_manager.create_xn_queue(exchange_name) try: xn.unbind(binding_key, xp) except TransportError: log.exception('Pubsub could not unbind %s from %s', binding_key, str(xp))
def stop(self): log.debug("Container stopping...") try: self.app_manager.stop() except Exception, ex: log.exception("Container stop(): Error stop AppManager")
def publish_event_object(self, event_object): """ Publishes an event of given type for the given origin. Event_type defaults to an event_type set when initializing the EventPublisher. Other kwargs fill out the fields of the event. This operation will fail with an exception. @param event_object the event object to be published @retval event_object the event object which was published """ assert event_object topic = self._topic(event_object) to_name = (self._send_name.exchange, topic) log.trace("Publishing event message to %s", to_name) current_time = int(get_ion_ts()) #Ensure valid created timestamp if supplied if event_object.ts_created: if not is_valid_ts(event_object.ts_created): raise BadRequest("The ts_created value is not a valid timestamp: '%s'" % (event_object.ts_created)) #Reject events that are older than specified time if int(event_object.ts_created) > ( current_time + VALID_EVENT_TIME_PERIOD ): raise BadRequest("This ts_created value is too far in the future:'%s'" % (event_object.ts_created)) #Reject events that are older than specified time if int(event_object.ts_created) < (current_time - VALID_EVENT_TIME_PERIOD) : raise BadRequest("This ts_created value is too old:'%s'" % (event_object.ts_created)) else: event_object.ts_created = str(current_time) #Validate this object #TODO - enable this once the resource agent issue sending a dict is figured out #event_object._validate() #Ensure the event object has a unique id if '_id' in event_object: raise BadRequest("The event object cannot contain a _id field '%s'" % (event_object)) #Generate a unique ID for this event event_object._id = create_unique_event_id() try: self.publish(event_object, to_name=to_name) except Exception as ex: log.exception("Failed to publish event (%s): '%s'" % (ex.message, event_object)) raise try: # store published event but only if we specified an event_repo if PERSIST_ON_PUBLISH and self.event_repo: self.event_repo.put_event(event_object) except Exception as ex: log.exception("Failed to store published event (%s): '%s'" % (ex.message, event_object)) raise return event_object
def register_dap_dataset(self, dataset_id, data_product_name=''): coverage_path = DatasetManagementService._get_coverage_path(dataset_id) try: self.add_dataset_to_xml(coverage_path=coverage_path, product_name=data_product_name) self.create_symlink(coverage_path, self.pydap_data_path) except: # We don't re-raise to prevent clients from bombing out... log.exception('Problem registering dataset') log.error('Failed to register dataset for coverage path %s' % coverage_path)
def call_process(self, packet, headers=None): try: self.process(packet) except Exception as e: log.exception('Unhandled caught in transform process') event_publisher = EventPublisher() event_publisher.publish_event(origin=self._transform_id, event_type='ExceptionEvent', exception_type=str(type(e)), exception_message=e.message)
def register_safe(self, parent, key, **kwargs): """ Use this method to protect caller from any form of directory register error """ try: return self.register(parent, key, **kwargs) except Exception as ex: log.exception("Error registering key=%s/%s, args=%s" % (parent, key, kwargs))
def delete_attachment(attachment_id): try: rr_client = ResourceRegistryServiceProcessClient(node=Container.instance.node, process=service_gateway_instance) ret = rr_client.delete_attachment(attachment_id) return gateway_json_response(ret) except Exception, e: log.exception("Error deleting attachment") return build_error_response(e)
def stop(self): log.debug("DatastoreManager.stop() [%d datastores]", len(self._datastores)) for x in self._datastores.itervalues(): try: x.close() except Exception as ex: log.exception("Error closing datastore") self._datastores = {}
def serve_forever(self): """ Run the container until killed. """ log.debug("In Container.serve_forever") if not self.proc_manager.proc_sup.running: self.start() # Exit if immediate==True and children len is ok num_procs = len(self.proc_manager.proc_sup.children) immediate = CFG.system.get('immediate', False) if immediate and num_procs == 1: # only spawned greenlet is the CC-Agent log.debug( "Container.serve_forever exiting due to CFG.system.immediate") else: # print a warning just in case if immediate and num_procs != 1: log.warn( "CFG.system.immediate=True but number of spawned processes is not 1 (%d)", num_procs) try: # This just waits in this Greenlet for all child processes to complete, # which is triggered somewhere else. self.proc_manager.proc_sup.join_children() except (KeyboardInterrupt, SystemExit) as ex: if hasattr(self, 'gl_parent_watch' ) and self.gl_parent_watch is not None: # Remove the greenlet that watches the parent process self.gl_parent_watch.kill() # Let the caller handle this raise except: log.exception('Unhandled error! Forcing container shutdown')
def stop(self): log.info("=============== Container stopping... ===============") self.event_pub.publish_event(event_type="ContainerLifecycleEvent", origin=self.id, origin_type="CapabilityContainer", sub_type="TERMINATE", state=ContainerStateEnum.TERMINATE) while self._capabilities: capability = self._capabilities.pop() log.debug("stop(): Stopping '%s'" % capability) try: self._stop_capability(capability) except Exception as ex: log.exception("Container stop(): Error stop %s" % capability) Container.instance = None from pyon.core import bootstrap bootstrap.container_instance = None self._is_started = False log.debug("Container stopped, OK.")
def on_start(self): try: SimpleProcess.on_start(self) self.pydap_host = self.CFG.get_safe( 'container.pydap_gateway.web_server.host', 'localhost') self.pydap_port = self.CFG.get_safe( 'container.pydap_gateway.web_server.port', '8001') self.pydap_data_path = self.CFG.get_safe('server.pydap.data_path', 'RESOURCE:ext/pydap') self.pydap_data_path = FileSystem.get_extended_url( self.pydap_data_path) self.app = make_app(None, self.pydap_data_path, 'ion/core/static/templates/') self.log = getLogger('pydap') self.log.write = self.log.info self.server = WSGIServer((self.pydap_host, int(self.pydap_port)), self.app, log=self.log) self.server.start() except: log.exception('Unable to start PyDAP server') raise
def execute_retrieve(self): ''' execute_retrieve Executes a retrieval and returns the result as a value in lieu of publishing it on a stream ''' try: coverage = DatasetManagementService._get_coverage(self.dataset_id,mode='r') if coverage.is_empty(): log.info('Reading from an empty coverage') rdt = RecordDictionaryTool(param_dictionary=coverage.parameter_dictionary) else: rdt = ReplayProcess._cov2granule(coverage=coverage, start_time=self.start_time, end_time=self.end_time, stride_time=self.stride_time, parameters=self.parameters, stream_def_id=self.delivery_format, tdoa=self.tdoa) except: log.exception('Problems reading from the coverage') raise BadRequest('Problems reading from the coverage') finally: coverage.close(timeout=5) return rdt.to_granule()
def get_event_computed_attributes(event, include_event=False, include_special=False, include_formatted=False): """ @param event any Event to compute attributes for @retval an EventComputedAttributes object for given event """ evt_computed = IonObject(OT.EventComputedAttributes) evt_computed.event_id = event._id evt_computed.ts_computed = get_ion_ts() evt_computed.event = event if include_event else None try: summary = get_event_summary(event) evt_computed.event_summary = summary if include_special: spc_attrs = [ "%s:%s" % (k, str(getattr(event, k))[:50]) for k in sorted(event.__dict__.keys()) if k not in [ '_id', '_rev', 'type_', 'origin', 'origin_type', 'ts_created', 'base_types' ] ] evt_computed.special_attributes = ", ".join(spc_attrs) if include_formatted: evt_computed.event_attributes_formatted = pprint.pformat( event.__dict__) except Exception as ex: log.exception( "Error computing EventComputedAttributes for event %s: %s", event, ex) return evt_computed
def _publish_event(self, event_msg, origin, event_type=None): event_type = event_type or self.event_type or event_msg._get_type() assert origin and event_type topic = self._topic(event_type, origin, base_types=event_msg.base_types, sub_type=event_msg.sub_type, origin_type=event_msg.origin_type) to_name = (self._send_name.exchange, topic) log.trace("Publishing event message to %s", to_name) try: self.publish(event_msg, to_name=to_name) except Exception as ex: log.exception("Failed to publish event '%s'" % (event_msg)) return False try: # store published event but only if we specified an event_repo if PERSIST_ON_PUBLISH and self.event_repo: self.event_repo.put_event(event_msg) except Exception as ex: log.exception("Failed to store published event '%s'" % (event_msg)) return False return True
def _load_actor(self, actor): ''' Returns callable execute method if it exists, otherwise it raises a BadRequest ''' try: module = __import__(actor['module'], fromlist=['']) except ImportError: log.exception('Actor could not be loaded') raise try: cls = getattr(module, actor['class']) except AttributeError: log.exception('Module %s does not have class %s', repr(module), actor['class']) raise try: execute = getattr(cls, 'execute') except AttributeError: log.exception('Actor class does not contain execute method') raise return execute
def validation(self, conditional, message, lineno): if not conditional: log.name = self.name log.exception('[%s] %s: %s', lineno or '?', self.exception.__name__, message) raise self.exception(message)
def spawn_process(self, name=None, module=None, cls=None, config=None, process_id=None): """ Spawn a process within the container. Processes can be of different type. """ if process_id and not is_valid_identifier(process_id, ws_sub='_'): raise BadRequest("Given process_id %s is not a valid identifier" % process_id) # Generate a new process id if not provided # TODO: Ensure it is system-wide unique process_id = process_id or "%s.%s" % (self.container.id, self.proc_id_pool.get_id()) log.debug( "ProcManager.spawn_process(name=%s, module.cls=%s.%s) as pid=%s", name, module, cls, process_id) if not config: # Use system CFG. It has the command line args in it config = DictModifier(CFG) else: # Use provided config. Must be dict or DotDict if not isinstance(config, DotDict): config = DotDict(config) config = DictModifier(CFG, config) if self.container.spawn_args: # Override config with spawn args dict_merge(config, self.container.spawn_args, inplace=True) #log.debug("spawn_process() pid=%s config=%s", process_id, config) # PROCESS TYPE. Determines basic process context (messaging, service interface) # One of: service, stream_process, agent, simple, immediate service_cls = named_any("%s.%s" % (module, cls)) process_type = get_safe(config, "process.type") or getattr( service_cls, "process_type", "service") service_instance = None try: # spawn service by type if process_type == "service": service_instance = self._spawn_service_process( process_id, name, module, cls, config) elif process_type == "stream_process": service_instance = self._spawn_stream_process( process_id, name, module, cls, config) elif process_type == "agent": service_instance = self._spawn_agent_process( process_id, name, module, cls, config) elif process_type == "standalone": service_instance = self._spawn_standalone_process( process_id, name, module, cls, config) elif process_type == "immediate": service_instance = self._spawn_immediate_process( process_id, name, module, cls, config) elif process_type == "simple": service_instance = self._spawn_simple_process( process_id, name, module, cls, config) else: raise BadRequest("Unknown process type: %s" % process_type) service_instance._proc_type = process_type self._register_process(service_instance, name) service_instance.errcause = "OK" log.info("AppManager.spawn_process: %s.%s -> pid=%s OK", module, cls, process_id) if process_type == 'immediate': log.info('Terminating immediate process: %s', service_instance.id) self.terminate_process(service_instance.id) return service_instance.id except Exception: errcause = service_instance.errcause if service_instance else "instantiating service" log.exception("Error spawning %s %s process (process_id: %s): %s", name, process_type, process_id, errcause) raise
def _control_flow(self): """ Entry point for process control thread of execution. This method is run by the control greenlet for each ION process. Listeners attached to the process, either RPC Servers or Subscribers, synchronize calls to the process by placing call requests into the queue by calling _routing_call. This method blocks until there are calls to be made in the synchronized queue, and then calls from within this greenlet. Any exception raised is caught and re-raised in the greenlet that originally scheduled the call. If successful, the AsyncResult created at scheduling time is set with the result of the call. """ svc_name = getattr( self.service, "name", "unnamed-service") if self.service else "unnamed-service" proc_id = getattr(self.service, "id", "unknown-pid") if self.service else "unknown-pid" if self.name: threading.current_thread().name = "%s-%s" % (svc_name, self.name) thread_base_name = threading.current_thread().name self._ready_control.set() for calltuple in self._ctrl_queue: calling_gl, ar, call, callargs, callkwargs, context = calltuple request_id = (context or {}).get("request-id", None) if request_id: threading.current_thread( ).name = thread_base_name + "-" + str(request_id) #log.debug("control_flow making call: %s %s %s (has context: %s)", call, callargs, callkwargs, context is not None) res = None start_proc_time = get_ion_ts_millis() self._record_proc_time(start_proc_time) # check context for expiration if context is not None and 'reply-by' in context: if start_proc_time >= int(context['reply-by']): log.info( "control_flow: attempting to process message already exceeding reply-by, ignore" ) # raise a timeout in the calling thread to allow endpoints to continue processing e = IonTimeout( "Reply-by time has already occurred (reply-by: %s, op start time: %s)" % (context['reply-by'], start_proc_time)) calling_gl.kill(exception=e, block=False) continue # If ar is set, means it is cancelled if ar.ready(): log.info( "control_flow: attempting to process message that has been cancelled, ignore" ) continue init_db_stats() try: # ****************************************************************** # ****** THIS IS WHERE THE RPC OPERATION/SERVICE CALL IS MADE ****** with self.service.push_context(context), \ self.service.container.context.push_context(context): self._ctrl_current = ar res = call(*callargs, **callkwargs) # ****** END CALL, EXCEPTION HANDLING FOLLOWS ****** # ****************************************************************** except OperationInterruptedException: # endpoint layer takes care of response as it's the one that caused this log.debug("Operation interrupted") pass except Exception as e: if self._log_call_exception: log.exception("PROCESS exception: %s" % e.message) # Raise the exception in the calling greenlet. # Try decorating the args of the exception with the true traceback - # this should be reported by ThreadManager._child_failed exc = PyonThreadTraceback( "IonProcessThread _control_flow caught an exception " "(call: %s, *args %s, **kwargs %s, context %s)\n" "True traceback captured by IonProcessThread' _control_flow:\n\n%s" % (call, callargs, callkwargs, context, traceback.format_exc())) e.args = e.args + (exc, ) if isinstance(e, (TypeError, IonException)): # Pass through known process exceptions, in particular IonException calling_gl.kill(exception=e, block=False) else: # Otherwise, wrap unknown, forward and hopefully we can continue on our way self._errors.append( (call, callargs, callkwargs, context, e, exc)) log.warn(exc) log.warn("Attempting to continue...") # Note: Too large exception string will crash the container (when passed on as msg header). exception_str = str(exc) if len(exception_str) > 10000: exception_str = ( "Exception string representation too large. " "Begin and end of the exception:\n" + exception_str[:2000] + "\n...\n" + exception_str[-2000:]) calling_gl.kill(exception=ContainerError(exception_str), block=False) finally: try: # Compute statistics self._compute_proc_stats(start_proc_time) db_stats = get_db_stats() if db_stats: if self._warn_call_dbstmt_threshold > 0 and db_stats.get( "count.all", 0) >= self._warn_call_dbstmt_threshold: stats_str = ", ".join( "{}={}".format(k, db_stats[k]) for k in sorted(db_stats.keys())) log.warn( "PROC_OP '%s.%s' EXCEEDED DB THRESHOLD. stats=%s", svc_name, call.__name__, stats_str) elif self._log_call_dbstats: stats_str = ", ".join( "{}={}".format(k, db_stats[k]) for k in sorted(db_stats.keys())) log.info("PROC_OP '%s.%s' DB STATS: %s", svc_name, call.__name__, stats_str) clear_db_stats() if stats_callback: stats_callback(proc_id=proc_id, proc_name=self.name, svc=svc_name, op=call.__name__, request_id=request_id, context=context, db_stats=db_stats, proc_stats=self.time_stats, result=res, exc=None) except Exception: log.exception("Error computing process call stats") self._ctrl_current = None threading.current_thread().name = thread_base_name # Set response in AsyncEvent of caller (endpoint greenlet) ar.set(res)
def test_stacks(self): try: self.throw_caused() except: log.exception('ion caused by python')
except Exception, ex: log.exception("Appl %s start from processapp failed" % appdef.name) else: # Case 2: Appdef contains full app start params modpath = appdef.mod try: mod = named_any(modpath) appdef._mod_loaded = mod # Start the app supid, state = mod.start(self.container, START_PERMANENT, appdef, config) appdef._supid = supid appdef._state = state log.debug("App '%s' started. Root sup-id=%s" % (appdef.name, supid)) self.apps.append(appdef) except Exception, ex: log.exception("Appl %s start from appdef failed" % appdef.name) def stop_app(self, appdef): log.debug("App '%s' stopping" % appdef.name) try: if '_mod_loaded' in appdef: appdef._mod_loaded.stop(self.container, appdef._state) except Exception, ex: log.exception("Application %s stop failed" % appdef.name)
def spawn_process(self, name=None, module=None, cls=None, config=None, process_id=None): """ Spawn a process within the container. Processes can be of different type. """ if process_id and not is_valid_identifier(process_id, ws_sub='_'): raise BadRequest("Given process_id %s is not a valid identifier" % process_id) # Generate a new process id if not provided # TODO: Ensure it is system-wide unique process_id = process_id or "%s.%s" % (self.container.id, self.proc_id_pool.get_id()) log.debug( "ProcManager.spawn_process(name=%s, module.cls=%s.%s, config=%s) as pid=%s", name, module, cls, config, process_id) process_cfg = deepcopy(CFG) if config: # Use provided config. Must be dict or DotDict if not isinstance(config, DotDict): config = DotDict(config) dict_merge(process_cfg, config, inplace=True) if self.container.spawn_args: # Override config with spawn args dict_merge(process_cfg, self.container.spawn_args, inplace=True) #log.debug("spawn_process() pid=%s process_cfg=%s", process_id, process_cfg) # PROCESS TYPE. Determines basic process context (messaging, service interface) # One of: service, stream_process, agent, simple, immediate service_cls = named_any("%s.%s" % (module, cls)) process_type = get_safe(process_cfg, "process.type") or getattr( service_cls, "process_type", "service") process_start_mode = get_safe(config, "process.start_mode") process_instance = None # alert we have a spawning process, but we don't have the instance yet, so give the class instead (more accurate than name) self._call_proc_state_changed("%s.%s" % (module, cls), ProcessStateEnum.PENDING) try: # spawn service by type if process_type == "service": process_instance = self._spawn_service_process( process_id, name, module, cls, process_cfg) elif process_type == "stream_process": process_instance = self._spawn_stream_process( process_id, name, module, cls, process_cfg) elif process_type == "agent": process_instance = self._spawn_agent_process( process_id, name, module, cls, process_cfg) elif process_type == "standalone": process_instance = self._spawn_standalone_process( process_id, name, module, cls, process_cfg) elif process_type == "immediate": process_instance = self._spawn_immediate_process( process_id, name, module, cls, process_cfg) elif process_type == "simple": process_instance = self._spawn_simple_process( process_id, name, module, cls, process_cfg) else: raise BadRequest("Unknown process type: %s" % process_type) process_instance._proc_type = process_type self._register_process(process_instance, name) process_instance.errcause = "OK" log.info("ProcManager.spawn_process: %s.%s -> pid=%s OK", module, cls, process_id) if process_type == 'immediate': log.info('Terminating immediate process: %s', process_instance.id) self.terminate_process(process_instance.id) # terminate process also triggers TERMINATING/TERMINATED self._call_proc_state_changed(process_instance, ProcessStateEnum.EXITED) else: #Shouldn't be any policies for immediate processes self.update_container_policies(process_instance) return process_instance.id except IonProcessError: errcause = process_instance.errcause if process_instance else "instantiating process" log.exception("Error spawning %s %s process (process_id: %s): %s", name, process_type, process_id, errcause) return None except Exception: errcause = process_instance.errcause if process_instance else "instantiating process" log.exception("Error spawning %s %s process (process_id: %s): %s", name, process_type, process_id, errcause) # trigger failed notification - catches problems in init/start self._call_proc_state_changed(process_instance, ProcessStateEnum.FAILED) raise
def register_safe(self, parent, key, **kwargs): try: return self.register(parent, key, **kwargs) except Exception as ex: log.exception("Error registering key=%s/%s, args=%s" % (parent, key, kwargs))
def unregister_safe(self, parent, key): try: return self.unregister(parent, key) except Exception as ex: log.exception("Error unregistering key=%s/%s" % (parent, key))
def publish_event_object(self, event_object): """ Publishes an event of given type for the given origin. Event_type defaults to an event_type set when initializing the EventPublisher. Other kwargs fill out the fields of the event. This operation will fail with an exception. @param event_object the event object to be published @retval event_object the event object which was published """ assert event_object topic = self._topic(event_object) to_name = (self._send_name.exchange, topic) log.trace("Publishing event message to %s", to_name) current_time = int(get_ion_ts()) #Ensure valid created timestamp if supplied if event_object.ts_created: if not is_valid_ts(event_object.ts_created): raise BadRequest( "The ts_created value is not a valid timestamp: '%s'" % (event_object.ts_created)) #Reject events that are older than specified time if int(event_object.ts_created) > (current_time + VALID_EVENT_TIME_PERIOD): raise BadRequest( "This ts_created value is too far in the future:'%s'" % (event_object.ts_created)) #Reject events that are older than specified time if int(event_object.ts_created) < (current_time - VALID_EVENT_TIME_PERIOD): raise BadRequest("This ts_created value is too old:'%s'" % (event_object.ts_created)) else: event_object.ts_created = str(current_time) #Validate this object #TODO - enable this once the resource agent issue sending a dict is figured out #event_object._validate() #Ensure the event object has a unique id if '_id' in event_object: raise BadRequest( "The event object cannot contain a _id field '%s'" % (event_object)) #Generate a unique ID for this event event_object._id = create_unique_event_id() try: self.publish(event_object, to_name=to_name) except Exception as ex: log.exception("Failed to publish event (%s): '%s'" % (ex.message, event_object)) raise try: # store published event but only if we specified an event_repo if PERSIST_ON_PUBLISH and self.event_repo: self.event_repo.put_event(event_object) except Exception as ex: log.exception("Failed to store published event (%s): '%s'" % (ex.message, event_object)) raise return event_object
def publish_event_object(self, event_object): """ Publishes an event of given type for the given origin. Event_type defaults to an event_type set when initializing the EventPublisher. Other kwargs fill out the fields of the event. This operation will fail with an exception. @param event_object the event object to be published @retval event_object the event object which was published """ if not event_object: raise BadRequest("Must provide event_object") event_object.base_types = event_object._get_extends() topic = self._topic( event_object ) # Routing key generated using type_, base_types, origin, origin_type, sub_type container = (hasattr(self, '_process') and hasattr( self._process, 'container') and self._process.container ) or BaseEndpoint._get_container_instance() if container and container.has_capability( container.CCAP.EXCHANGE_MANAGER): # make sure we are an xp, if not, upgrade if not isinstance(self._send_name, XOTransport): default_nt = NameTrio(self.get_events_exchange_point()) if isinstance(self._send_name, NameTrio) \ and self._send_name.exchange == default_nt.exchange \ and self._send_name.queue == default_nt.queue \ and self._send_name.binding == default_nt.binding: self._send_name = container.create_xp(self._events_xp) else: self._send_name = container.create_xp(self._send_name) xp = self._send_name to_name = xp.create_route(topic) else: to_name = (self._send_name.exchange, topic) current_time = get_ion_ts_millis() # Ensure valid created timestamp if supplied if event_object.ts_created: if not is_valid_ts(event_object.ts_created): raise BadRequest( "The ts_created value is not a valid timestamp: '%s'" % (event_object.ts_created)) # Reject events that are older than specified time if int(event_object.ts_created) > (current_time + VALID_EVENT_TIME_PERIOD): raise BadRequest( "This ts_created value is too far in the future:'%s'" % (event_object.ts_created)) # Reject events that are older than specified time if int(event_object.ts_created) < (current_time - VALID_EVENT_TIME_PERIOD): raise BadRequest("This ts_created value is too old:'%s'" % (event_object.ts_created)) else: event_object.ts_created = str(current_time) # Set the actor id based on if not event_object.actor_id: event_object.actor_id = self._get_actor_id() #Validate this object - ideally the validator should pass on problems, but for now just log #any errors and keep going, since seeing invalid situations are better than skipping validation. try: event_object._validate() except Exception as e: log.exception(e) #Ensure the event object has a unique id if '_id' in event_object: raise BadRequest( "The event object cannot contain a _id field '%s'" % (event_object)) #Generate a unique ID for this event event_object._id = create_unique_event_id() try: self.publish(event_object, to_name=to_name) except Exception as ex: log.exception("Failed to publish event (%s): '%s'" % (ex.message, event_object)) raise return event_object
def get_dataset(self, cov, fields, fill_index, dataset, response): for var in fields: while var: name, slice_ = var.pop(0) name = urllib.unquote(name) slice_ = self.update_slice_object(slice_, fill_index) if slice_ is None: continue #print name #print "slice", slice_ pc = cov.get_parameter_context(name) try: param = cov.get_parameter(name) data = np.array([]) time_data = np.array([]) if response == "dods": data = self.get_data(cov, name, slice_) time_data = self.get_time_data(cov, slice_) time_attrs = self.get_attrs(cov, name) attrs = self.get_attrs(cov, name) dims = (cov.temporal_parameter_name,) if isinstance(pc.param_type, QuantityType) and not param.is_coordinate and cov.temporal_parameter_name != name: dataset[name] = self.make_grid(name, data, time_data, attrs, time_attrs, dims, data.dtype.char) if isinstance(pc.param_type, ConstantType): dataset[name] = self.make_grid(name, data, time_data, attrs, time_attrs, dims, data.dtype.char) if isinstance(pc.param_type, ConstantRangeType): #start = time.time() #convert to string try: #scalar case if data.shape == (2,): data = np.atleast_1d('_'.join([str(data[0]), str(data[1])])) else: for i,d in enumerate(data): f = [str(d[0]),str(d[1])] data[i] = '_'.join(f) except Exception, e: data = np.asanyarray(['None' for d in data]) #print "range end", time.time() - start dataset[name] = self.make_grid(name, data, time_data, attrs, time_attrs, dims, 'S') if isinstance(pc.param_type,BooleanType): data = np.asanyarray(data, dtype='int32') dataset[name] = self.make_grid(name, data, time_data, attrs, time_attrs, dims, data.dtype.char) if isinstance(pc.param_type,CategoryType): #start = time.time() dataset[name] = self.make_grid(name, data, time_data, attrs, time_attrs, dims, self.get_numpy_type(data)) #print "category end", time.time() - start if isinstance(pc.param_type,ArrayType): #start = time.time() dataset[name] = self.make_grid(name, data, time_data, attrs, time_attrs, dims, self.get_numpy_type(data)) #print "array end", time.time() - start if isinstance(pc.param_type,RecordType): #start = time.time() #convert to string try: for i,ddict in enumerate(data): data[i] = str(ddict) except Exception, e: data = np.asanyarray(['None' for d in data]) #print "record end", time.time() - start dataset[name] = self.make_grid(name, data, time_data, attrs, time_attrs, dims, 'S') if param.is_coordinate and cov.temporal_parameter_name == name: dataset[name] = BaseType(name=name, data=data, type=data.dtype.char, attributes=attrs, shape=data.shape) except Exception, e: log.exception('Problem reading cov %s %s', cov.name, e) continue
def acquire_lock(self, key, timeout=LOCK_EXPIRES_DEFAULT, lock_holder=None, lock_info=None): """ Attempts to atomically acquire a lock with the given key and namespace. If holder is given and holder already has the lock, renew. Checks for expired locks. @param timeout Secs until lock expiration or 0 for no expiration @param lock_holder Str value identifying lock holder for subsequent exclusive access @param lock_info Dict value for additional attributes describing lock @retval bool - could lock be acquired? """ if not key: raise BadRequest("Missing argument: key") if "/" in key: raise BadRequest("Invalid argument value: key") lock_attrs = {LOCK_EXPIRES_ATTR: get_ion_ts_millis() + int(1000*timeout) if timeout else 0, LOCK_HOLDER_ATTR: lock_holder or ""} if lock_info: lock_attrs.update(lock_info) expires = int(lock_attrs[LOCK_EXPIRES_ATTR]) # Check type just to be sure if expires and get_ion_ts_millis() > expires: raise BadRequest("Invalid lock expiration value: %s", expires) direntry = self._create_dir_entry(LOCK_DIR_PATH, key, attributes=lock_attrs) lock_result = False try: # This is an atomic operation. It relies on the unique key constraint of the directory service self.dir_store.create(direntry, create_unique_directory_id()) lock_result = True except BadRequest as ex: if ex.message.startswith("DirEntry already exists"): de_old = self.lookup(LOCK_DIR_PATH, key, return_entry=True) if de_old: if self._is_lock_expired(de_old): # Lock is expired: remove, try to relock # Note: even as holder, it's safer to reacquire in this case than renew log.warn("Removing expired lock: %s/%s", de_old.parent, de_old.key) try: # This is safe, because of lock was deleted + recreated in the meantime, it has different id self._delete_lock(de_old) # Try recreate - may fail again due to concurrency self.dir_store.create(direntry, create_unique_directory_id()) lock_result = True except BadRequest as ex: if not ex.message.startswith("DirEntry already exists"): log.exception("Error releasing/reacquiring expired lock %s", de_old.key) except Exception: log.exception("Error releasing/reacquiring expired lock %s", de_old.key) elif lock_holder and de_old.attributes[LOCK_HOLDER_ATTR] == lock_holder: # Holder currently holds the lock: renew log.debug("Renewing lock %s/%s for holder %s", de_old.parent, de_old.key, lock_holder) de_old.attributes = lock_attrs try: self.dir_store.update(de_old) lock_result = True except Exception: log.exception("Error renewing expired lock %s", de_old.key) # We do nothing if we could not find the lock now... else: raise log.debug("Directory.acquire_lock(%s): %s -> %s", key, lock_attrs, lock_result) return lock_result
dtype) elif isinstance(pc.param_type, SparseConstantType): data, dtype = self.filter_data(data) seq[name] = self.make_series(response, name, data, attrs, dtype) #dataset[name] = self.make_series(response, name, data, attrs, dtype) # elif param.is_coordinate and cov.temporal_parameter_name == name: # dataset[name] = BaseType(name=name, data=data, type=data.dtype.char, attributes=attrs, shape=data.shape) # else: # log.error("Unhandled parameter for parameter (%s) type: %s", name, pc.param_type.__class__.__name__) except Exception, e: log.exception('Problem reading cov %s %s', cov.name, e.__class__.__name__) continue dataset['data'] = seq return dataset def value_encoding_to_dap_type(self, value_encoding): if value_encoding is None: return 'S' dt = np.dtype(value_encoding).char if dt == 'O': return 'S' return dt def dap_type(self, context): if isinstance(
def wrapper(*args, **kwargs): try: return func(*args, **kwargs) except: log.exception("Failed handling PyDAP request") raise
class ProcManager(object): def __init__(self, container): self.container = container # Define the callables that can be added to Container public API self.container_api = [self.spawn_process, self.terminate_process] # Add the public callables to Container for call in self.container_api: setattr(self.container, call.__name__, call) self.proc_id_pool = IDPool() # Temporary registry of running processes self.procs_by_name = {} self.procs = {} # mapping of greenlets we spawn to process_instances for error handling self._spawned_proc_to_process = {} # The pyon worker process supervisor self.proc_sup = IonProcessThreadManager( heartbeat_secs=CFG.cc.timeout.heartbeat, failure_notify_callback=self._spawned_proc_failed) # list of callbacks for process state changes self._proc_state_change_callbacks = [] def start(self): log.debug("ProcManager starting ...") self.proc_sup.start() # Register container as resource object cc_obj = CapabilityContainer(name=self.container.id, cc_agent=self.container.name) self.cc_id, _ = self.container.resource_registry.create(cc_obj) #Create an association to an Org object if not the rot ION org and only if found if CFG.container.org_name and CFG.container.org_name != CFG.system.root_org: org, _ = self.container.resource_registry.find_resources( restype=RT.Org, name=CFG.container.org_name, id_only=True) if org: self.container.resource_registry.create_association( org[0], PRED.hasResource, self.cc_id) # TODO - replace with proper association log.debug("ProcManager started, OK.") def stop(self): log.debug("ProcManager stopping ...") from pyon.datastore.couchdb.couchdb_datastore import CouchDB_DataStore stats1 = CouchDB_DataStore._stats.get_stats() # Call quit on procs to give them ability to clean up # @TODO terminate_process is not gl-safe # gls = map(lambda k: spawn(self.terminate_process, k), self.procs.keys()) # join(gls) procs_list = sorted(self.procs.values(), key=lambda proc: proc._proc_start_time, reverse=True) for proc in procs_list: try: self.terminate_process(proc.id) except Exception as ex: log.warn("Failed to terminate process (%s): %s", proc.id, ex) # TODO: Have a choice of shutdown behaviors for waiting on children, timeouts, etc self.proc_sup.shutdown(CFG.cc.timeout.shutdown) if self.procs: log.warn("ProcManager procs not empty: %s", self.procs) if self.procs_by_name: log.warn("ProcManager procs_by_name not empty: %s", self.procs_by_name) # Remove Resource registration try: self.container.resource_registry.delete(self.cc_id, del_associations=True) except NotFound: # already gone, this is ok pass # TODO: Check associations to processes stats2 = CouchDB_DataStore._stats.get_stats() stats3 = CouchDB_DataStore._stats.diff_stats(stats2, stats1) log.debug("Datastore stats difference during stop(): %s", stats3) log.debug("ProcManager stopped, OK.") def spawn_process(self, name=None, module=None, cls=None, config=None, process_id=None): """ Spawn a process within the container. Processes can be of different type. """ if process_id and not is_valid_identifier(process_id, ws_sub='_'): raise BadRequest("Given process_id %s is not a valid identifier" % process_id) # Generate a new process id if not provided # TODO: Ensure it is system-wide unique process_id = process_id or "%s.%s" % (self.container.id, self.proc_id_pool.get_id()) log.debug( "ProcManager.spawn_process(name=%s, module.cls=%s.%s, config=%s) as pid=%s", name, module, cls, config, process_id) process_cfg = deepcopy(CFG) if config: # Use provided config. Must be dict or DotDict if not isinstance(config, DotDict): config = DotDict(config) dict_merge(process_cfg, config, inplace=True) if self.container.spawn_args: # Override config with spawn args dict_merge(process_cfg, self.container.spawn_args, inplace=True) #log.debug("spawn_process() pid=%s process_cfg=%s", process_id, process_cfg) # PROCESS TYPE. Determines basic process context (messaging, service interface) # One of: service, stream_process, agent, simple, immediate service_cls = named_any("%s.%s" % (module, cls)) process_type = get_safe(process_cfg, "process.type") or getattr( service_cls, "process_type", "service") process_start_mode = get_safe(config, "process.start_mode") process_instance = None # alert we have a spawning process, but we don't have the instance yet, so give the class instead (more accurate than name) self._call_proc_state_changed("%s.%s" % (module, cls), ProcessStateEnum.PENDING) try: # spawn service by type if process_type == "service": process_instance = self._spawn_service_process( process_id, name, module, cls, process_cfg) elif process_type == "stream_process": process_instance = self._spawn_stream_process( process_id, name, module, cls, process_cfg) elif process_type == "agent": process_instance = self._spawn_agent_process( process_id, name, module, cls, process_cfg) elif process_type == "standalone": process_instance = self._spawn_standalone_process( process_id, name, module, cls, process_cfg) elif process_type == "immediate": process_instance = self._spawn_immediate_process( process_id, name, module, cls, process_cfg) elif process_type == "simple": process_instance = self._spawn_simple_process( process_id, name, module, cls, process_cfg) else: raise BadRequest("Unknown process type: %s" % process_type) process_instance._proc_type = process_type self._register_process(process_instance, name) process_instance.errcause = "OK" log.info("ProcManager.spawn_process: %s.%s -> pid=%s OK", module, cls, process_id) if process_type == 'immediate': log.info('Terminating immediate process: %s', process_instance.id) self.terminate_process(process_instance.id) # terminate process also triggers TERMINATING/TERMINATED self._call_proc_state_changed(process_instance, ProcessStateEnum.EXITED) else: #Shouldn't be any policies for immediate processes self.update_container_policies(process_instance) return process_instance.id except IonProcessError: errcause = process_instance.errcause if process_instance else "instantiating process" log.exception("Error spawning %s %s process (process_id: %s): %s", name, process_type, process_id, errcause) return None except Exception: errcause = process_instance.errcause if process_instance else "instantiating process" log.exception("Error spawning %s %s process (process_id: %s): %s", name, process_type, process_id, errcause) # trigger failed notification - catches problems in init/start self._call_proc_state_changed(process_instance, ProcessStateEnum.FAILED) raise #This must be called after registering the process def update_container_policies(self, process_instance): if not self.container.governance_controller: return if process_instance._proc_type == "service": # look to load any existing policies for this service self.container.governance_controller.safe_update_service_access_policy( process_instance._proc_listen_name) if process_instance._proc_type == "agent": # look to load any existing policies for this agent service if process_instance.resource_type is None: self.container.governance_controller.safe_update_service_access_policy( process_instance.name) else: self.container.governance_controller.safe_update_service_access_policy( process_instance.resource_type) if process_instance.resource_id: # look to load any existing policies for this resource self.container.governance_controller.safe_update_resource_access_policy( process_instance.resource_id) def list_local_processes(self, process_type=''): """ Returns a list of the running ION processes in the container or filtered by the process_type """ if not process_type: return self.procs.values() return [ p for p in self.procs.itervalues() if p.process_type == process_type ] def get_a_local_process(self, proc_name=''): """ Returns a running ION processes in the container for the specified name """ for p in self.procs.itervalues(): if p.name == proc_name: return p if p.process_type == 'agent' and p.resource_type == proc_name: return p return None def is_local_service_process(self, service_name): local_services = self.list_local_processes('service') for p in local_services: if p.name == service_name: return True return False def is_local_agent_process(self, resource_type): local_agents = self.list_local_processes('agent') for p in local_agents: if p.resource_type == resource_type: return True return False def _spawned_proc_failed(self, gproc): log.error("ProcManager._spawned_proc_failed: %s, %s", gproc, gproc.exception) # for now - don't worry about the mapping, if we get a failure, just kill the container. # leave the mapping in place for potential expansion later. # # look it up in mapping # if not gproc in self._spawned_proc_to_process: # log.warn("No record of gproc %s in our map (%s)", gproc, self._spawned_proc_to_process) # return # prc = self._spawned_proc_to_process.get(gproc, None) # # # make sure prc is in our list # if not prc in self.procs.values(): # log.warn("prc %s not found in procs list", prc) # return # stop the rest of the process if prc is not None: try: self.terminate_process(prc.id, False) except Exception as e: log.warn( "Problem while stopping rest of failed process %s: %s", prc, e) finally: self._call_proc_state_changed(prc, ProcessStateEnum.FAILED) else: log.warn("No ION process found for failed proc manager child: %s", gproc) #self.container.fail_fast("Container process (%s) failed: %s" % (svc, gproc.exception)) def _cleanup_method(self, queue_name, ep=None): """ Common method to be passed to each spawned ION process to clean up their process-queue. @TODO Leaks implementation detail, should be using XOs """ if ep._chan is not None and not ep._chan._queue_auto_delete: # only need to delete if AMQP didn't handle it for us already! # @TODO this will not work with XOs (future) try: ch = self.container.node.channel(RecvChannel) ch._recv_name = NameTrio( get_sys_name(), "%s.%s" % (get_sys_name(), queue_name)) ch._destroy_queue() except TransportError as ex: log.warn("Cleanup method triggered an error, ignoring: %s", ex) def add_proc_state_changed_callback(self, cb): """ Adds a callback to be called when a process' state changes. The callback should take three parameters: The process, the state, and the container. """ self._proc_state_change_callbacks.append(cb) def remove_proc_state_changed_callback(self, cb): """ Removes a callback from the process state change callback list. If the callback is not registered, this method does nothing. """ if cb in self._proc_state_change_callbacks: self._proc_state_change_callbacks.remove(cb) def _call_proc_state_changed(self, svc, state): """ Internal method to call all registered process state change callbacks. """ log.debug("Proc State Changed (%s): %s", ProcessStateEnum._str_map.get(state, state), svc) for cb in self._proc_state_change_callbacks: cb(svc, state, self.container) def _create_listening_endpoint(self, **kwargs): """ Creates a listening endpoint for spawning processes. This method exists to be able to override the type created via configuration. In most cases it will create a ConversationRPCServer. """ eptypestr = CFG.get_safe( 'container.messaging.endpoint.proc_listening_type', None) if eptypestr is not None: module, cls = eptypestr.rsplit('.', 1) mod = __import__(module, fromlist=[cls]) eptype = getattr(mod, cls) ep = eptype(**kwargs) else: conv_enabled = CFG.get_safe( 'container.messaging.endpoint.rpc_conversation_enabled', False) if conv_enabled: ep = ConversationRPCServer(**kwargs) else: ep = ProcessRPCServer(**kwargs) return ep # ----------------------------------------------------------------- # PROCESS TYPE: service def _spawn_service_process(self, process_id, name, module, cls, config): """ Spawn a process acting as a service worker. Attach to service queue with service definition, attach to service pid """ process_instance = self._create_process_instance( process_id, name, module, cls, config) listen_name = get_safe(config, "process.listen_name") or process_instance.name log.debug("Service Process (%s) listen_name: %s", name, listen_name) process_instance._proc_listen_name = listen_name # Service RPC endpoint rsvc1 = self._create_listening_endpoint(node=self.container.node, from_name=listen_name, service=process_instance, process=process_instance) # Named local RPC endpoint rsvc2 = self._create_listening_endpoint(node=self.container.node, from_name=process_instance.id, service=process_instance, process=process_instance) # cleanup method to delete process queue cleanup = lambda _: self._cleanup_method(process_instance.id, rsvc2) # Start an ION process with the right kind of endpoint factory proc = self.proc_sup.spawn(name=process_instance.id, service=process_instance, listeners=[rsvc1, rsvc2], proc_name=process_instance._proc_name, cleanup_method=cleanup) self.proc_sup.ensure_ready( proc, "_spawn_service_process for %s" % ",".join( (listen_name, process_instance.id))) # map gproc to process_instance self._spawned_proc_to_process[proc.proc] = process_instance # set service's reference to process process_instance._process = proc self._process_init(process_instance) self._process_start(process_instance) try: proc.start_listeners() except IonProcessError: self._process_quit(process_instance) self._call_proc_state_changed(process_instance, ProcessStateEnum.FAILED) raise return process_instance # ----------------------------------------------------------------- # PROCESS TYPE: stream process def _spawn_stream_process(self, process_id, name, module, cls, config): """ Spawn a process acting as a data stream process. Attach to subscription queue with process function. """ process_instance = self._create_process_instance( process_id, name, module, cls, config) listen_name = get_safe(config, "process.listen_name") or name log.debug("Stream Process (%s) listen_name: %s", name, listen_name) process_instance._proc_listen_name = listen_name process_instance.stream_subscriber = StreamSubscriber( process=process_instance, exchange_name=listen_name, callback=process_instance.call_process) # Add publishers if any... publish_streams = get_safe(config, "process.publish_streams") pub_names = self._set_publisher_endpoints(process_instance, publish_streams) rsvc = self._create_listening_endpoint(node=self.container.node, from_name=process_instance.id, service=process_instance, process=process_instance) # cleanup method to delete process queue (@TODO: leaks a bit here - should use XOs) def cleanup(*args): self._cleanup_method(process_instance.id, rsvc) for name in pub_names: p = getattr(process_instance, name) p.close() proc = self.proc_sup.spawn( name=process_instance.id, service=process_instance, listeners=[rsvc, process_instance.stream_subscriber], proc_name=process_instance._proc_name, cleanup_method=cleanup) self.proc_sup.ensure_ready( proc, "_spawn_stream_process for %s" % process_instance._proc_name) # map gproc to process_instance self._spawned_proc_to_process[proc.proc] = process_instance # set service's reference to process process_instance._process = proc self._process_init(process_instance) self._process_start(process_instance) try: proc.start_listeners() except IonProcessError: self._process_quit(process_instance) self._call_proc_state_changed(process_instance, ProcessStateEnum.FAILED) raise return process_instance # ----------------------------------------------------------------- # PROCESS TYPE: agent def _spawn_agent_process(self, process_id, name, module, cls, config): """ Spawn a process acting as agent process. Attach to service pid. """ process_instance = self._create_process_instance( process_id, name, module, cls, config) if not isinstance(process_instance, ResourceAgent) and not isinstance( process_instance, SimpleResourceAgent): raise ContainerConfigError( "Agent process must extend ResourceAgent") listeners = [] # Set the resource ID if we get it through the config resource_id = get_safe(process_instance.CFG, "agent.resource_id") if resource_id: process_instance.resource_id = resource_id alistener = self._create_listening_endpoint( node=self.container.node, from_name=resource_id, service=process_instance, process=process_instance) listeners.append(alistener) rsvc = self._create_listening_endpoint(node=self.container.node, from_name=process_instance.id, service=process_instance, process=process_instance) listeners.append(rsvc) # cleanup method to delete process/agent queue (@TODO: leaks a bit here - should use XOs) def agent_cleanup(x): self._cleanup_method(process_instance.id, rsvc) if resource_id: pass #self._cleanup_method(resource_id, alistener) # disabled, it's probably not architecturally correct to delete this queue proc = self.proc_sup.spawn(name=process_instance.id, service=process_instance, listeners=listeners, proc_name=process_instance._proc_name, cleanup_method=agent_cleanup) self.proc_sup.ensure_ready( proc, "_spawn_agent_process for %s" % process_instance.id) # map gproc to process_instance self._spawned_proc_to_process[proc.proc] = process_instance # set service's reference to process process_instance._process = proc # Now call the on_init of the agent. self._process_init(process_instance) if not process_instance.resource_id: log.warn("New agent pid=%s has no resource_id set" % process_id) self._process_start(process_instance) try: proc.start_listeners() except IonProcessError: self._process_quit(process_instance) self._call_proc_state_changed(process_instance, ProcessStateEnum.FAILED) raise if not process_instance.resource_id: log.warn("Agent process id=%s does not define resource_id!!" % process_instance.id) return process_instance # ----------------------------------------------------------------- # PROCESS TYPE: standalone def _spawn_standalone_process(self, process_id, name, module, cls, config): """ Spawn a process acting as standalone process. Attach to service pid. """ process_instance = self._create_process_instance( process_id, name, module, cls, config) rsvc = self._create_listening_endpoint(node=self.container.node, from_name=process_instance.id, service=process_instance, process=process_instance) # Add publishers if any... publish_streams = get_safe(config, "process.publish_streams") pub_names = self._set_publisher_endpoints(process_instance, publish_streams) # cleanup method to delete process queue (@TODO: leaks a bit here - should use XOs) def cleanup(*args): self._cleanup_method(process_instance.id, rsvc) for name in pub_names: p = getattr(process_instance, name) p.close() proc = self.proc_sup.spawn(name=process_instance.id, service=process_instance, listeners=[rsvc], proc_name=process_instance._proc_name, cleanup_method=cleanup) self.proc_sup.ensure_ready( proc, "_spawn_standalone_process for %s" % process_instance.id) # map gproc to process_instance self._spawned_proc_to_process[proc.proc] = process_instance # set service's reference to process process_instance._process = proc self._process_init(process_instance) self._process_start(process_instance) try: proc.start_listeners() except IonProcessError: self._process_quit(process_instance) self._call_proc_state_changed(process_instance, ProcessStateEnum.FAILED) raise return process_instance # ----------------------------------------------------------------- # PROCESS TYPE: simple def _spawn_simple_process(self, process_id, name, module, cls, config): """ Spawn a process acting as simple process. No attachments. """ process_instance = self._create_process_instance( process_id, name, module, cls, config) # Add publishers if any... publish_streams = get_safe(config, "process.publish_streams") pub_names = self._set_publisher_endpoints(process_instance, publish_streams) # cleanup method to delete process queue (@TODO: leaks a bit here - should use XOs) def cleanup(*args): for name in pub_names: p = getattr(process_instance, name) p.close() proc = self.proc_sup.spawn(name=process_instance.id, service=process_instance, listeners=[], proc_name=process_instance._proc_name, cleanup_method=cleanup) self.proc_sup.ensure_ready( proc, "_spawn_simple_process for %s" % process_instance.id) self._process_init(process_instance) self._process_start(process_instance) return process_instance # ----------------------------------------------------------------- # PROCESS TYPE: immediate def _spawn_immediate_process(self, process_id, name, module, cls, config): """ Spawn a process acting as immediate one off process. No attachments. """ process_instance = self._create_process_instance( process_id, name, module, cls, config) self._process_init(process_instance) self._process_start(process_instance) return process_instance def _create_process_instance(self, process_id, name, module, cls, config): """ Creates an instance of a "service", be it a Service, Agent, Stream, etc. @rtype BaseService @return An instance of a "service" """ # SERVICE INSTANCE. process_instance = for_name(module, cls) if not isinstance(process_instance, BaseService): raise ContainerConfigError( "Instantiated service not a BaseService %r" % process_instance) # Prepare service instance process_instance.errcause = "" process_instance.id = process_id process_instance.container = self.container process_instance.CFG = config process_instance._proc_name = name process_instance._proc_start_time = time.time() #Unless the process has been started as part of another Org, default to the container Org or the ION Org if config.has_key('org_name'): process_instance.org_name = config['org_name'] else: process_instance.org_name = CFG.get_safe( 'container.org_name', CFG.get_safe('system.root_org', 'ION')) # Add stateful process operations if hasattr(process_instance, "_flush_state"): def _flush_state(): if not hasattr(process_instance, "_proc_state"): process_instance._proc_state = {} process_instance._proc_state_changed = False return process_instance.container.state_repository.put_state( process_instance.id, process_instance._proc_state) process_instance._proc_state_changed = False def _load_state(): if not hasattr(process_instance, "_proc_state"): process_instance._proc_state = {} try: new_state = process_instance.container.state_repository.get_state( process_instance.id) process_instance._proc_state.clear() process_instance._proc_state.update(new_state) process_instance._proc_state_changed = False except Exception as ex: log.warn("Process %s load state failed: %s", process_instance.id, str(ex)) process_instance._flush_state = _flush_state process_instance._load_state = _load_state process_start_mode = get_safe(config, "process.start_mode") if process_start_mode == "RESTART": if hasattr(process_instance, "_load_state"): process_instance._load_state() # start service dependencies (RPC clients) self._start_process_dependencies(process_instance) return process_instance def _start_process_dependencies(self, process_instance): process_instance.errcause = "setting service dependencies" log.debug("spawn_process dependencies: %s", process_instance.dependencies) # TODO: Service dependency != process dependency for dependency in process_instance.dependencies: client = getattr(process_instance.clients, dependency) assert client, "Client for dependency not found: %s" % dependency # @TODO: should be in a start_client in RPCClient chain client.process = process_instance client.node = self.container.node # ensure that dep actually exists and is running # MM: commented out - during startup (init actually), we don't need to check for service dependencies # MM: TODO: split on_init from on_start; start consumer in on_start; check for full queues on restart # if process_instance.name != 'bootstrap' or (process_instance.name == 'bootstrap' and process_instance.CFG.level == dependency): # svc_de = self.container.resource_registry.find_resources(restype="Service", name=dependency, id_only=True) # if not svc_de: # raise ContainerConfigError("Dependency for service %s not running: %s" % (process_instance.name, dependency)) def _process_init(self, process_instance): # Init process process_instance.errcause = "initializing service" process_instance.init() def _process_start(self, process_instance): # Start process # THIS SHOULD BE CALLED LATER THAN SPAWN # TODO: Check for timeout process_instance.errcause = "starting service" process_instance.start() def _process_quit(self, process_instance): """ Common method to handle process stopping. """ process_instance.errcause = "quitting process" # Give the process notice to quit doing stuff. process_instance.quit() # Terminate IonProcessThread (may not have one, i.e. simple process) # @TODO: move this into process' on_quit() if getattr(process_instance, '_process', None) is not None and process_instance._process: process_instance._process.notify_stop() process_instance._process.stop() def _set_publisher_endpoints(self, process_instance, publisher_streams=None): publisher_streams = publisher_streams or {} names = [] for name, stream_id in publisher_streams.iteritems(): # problem is here pub = StreamPublisher(process=process_instance, stream_id=stream_id) setattr(process_instance, name, pub) names.append(name) return names def _register_process(self, process_instance, name): """ Performs all actions related to registering the new process in the system. Also performs process type specific registration, such as for services and agents """ # Add process instance to container's process dict if name in self.procs_by_name: log.warn("Process name already registered in container: %s" % name) self.procs_by_name[name] = process_instance self.procs[process_instance.id] = process_instance # Add Process to resource registry # Note: In general the Process resource should be created by the CEI PD, but not all processes are CEI # processes. How to deal with this? process_instance.errcause = "registering" if process_instance._proc_type != "immediate": proc_obj = Process(name=process_instance.id, label=name, proctype=process_instance._proc_type) proc_id, _ = self.container.resource_registry.create(proc_obj) process_instance._proc_res_id = proc_id # Associate process with container resource self.container.resource_registry.create_association( self.cc_id, "hasProcess", proc_id) else: process_instance._proc_res_id = None # Process type specific registration # TODO: Factor out into type specific handler functions if process_instance._proc_type == "service": # Registration of SERVICE process: in resource registry service_list, _ = self.container.resource_registry.find_resources( restype="Service", name=process_instance.name) if service_list: process_instance._proc_svc_id = service_list[0]._id else: # We are starting the first process of a service instance # TODO: This should be created by the HA Service agent in the future svc_obj = Service( name=process_instance.name, exchange_name=process_instance._proc_listen_name, state=ServiceStateEnum.READY) process_instance._proc_svc_id, _ = self.container.resource_registry.create( svc_obj) # Create association to service definition resource svcdef_list, _ = self.container.resource_registry.find_resources( restype="ServiceDefinition", name=process_instance.name) if svcdef_list: self.container.resource_registry.create_association( process_instance._proc_svc_id, "hasServiceDefinition", svcdef_list[0]._id) else: log.error("Cannot find ServiceDefinition resource for %s", process_instance.name) self.container.resource_registry.create_association( process_instance._proc_svc_id, "hasProcess", proc_id) elif process_instance._proc_type == "agent": # Registration of AGENT process: in Directory caps = process_instance.get_capabilities() self.container.directory.register( "/Agents", process_instance.id, **dict(name=process_instance._proc_name, container=process_instance.container.id, resource_id=process_instance.resource_id, agent_id=process_instance.agent_id, def_id=process_instance.agent_def_id, capabilities=caps)) self._call_proc_state_changed(process_instance, ProcessStateEnum.RUNNING) def terminate_process(self, process_id, do_notifications=True): """ Terminates a process and all its resources. Termination is graceful with timeout. @param process_id The id of the process to terminate. Should exist in the container's list of processes or this will raise. @param do_notifications If True, emits process state changes for TERMINATING and TERMINATED. If False, supresses any state changes. Used near EXITED and FAILED. """ process_instance = self.procs.get(process_id, None) if not process_instance: raise BadRequest( "Cannot terminate. Process id='%s' unknown on container id='%s'" % (process_id, self.container.id)) log.info("ProcManager.terminate_process: %s -> pid=%s", process_instance._proc_name, process_id) if do_notifications: self._call_proc_state_changed(process_instance, ProcessStateEnum.TERMINATING) self._process_quit(process_instance) self._unregister_process(process_id, process_instance) if do_notifications: self._call_proc_state_changed(process_instance, ProcessStateEnum.TERMINATED) def _unregister_process(self, process_id, process_instance): # Remove process registration in resource registry if process_instance._proc_res_id: try: self.container.resource_registry.delete( process_instance._proc_res_id, del_associations=True) except NotFound: #, HTTPException): # if it's already gone, it's already gone! pass except Exception, ex: log.exception(ex) pass # Cleanup for specific process types if process_instance._proc_type == "service": # Check if this is the last process for this service and do auto delete service resources here svcproc_list = [] try: svcproc_list, _ = self.container.resource_registry.find_objects( process_instance._proc_svc_id, "hasProcess", "Process", id_only=True) except ResourceNotFound: # if it's already gone, it's already gone! pass if not svcproc_list: try: self.container.resource_registry.delete( process_instance._proc_svc_id, del_associations=True) except NotFound: # if it's already gone, it's already gone! pass except Exception, ex: log.exception(ex) pass
def gradient_test_parser(document): ''' This parser YIELDS a document per call or until it's done The format for the document is CSV in this table format Array,Instrument Class,Reference Designator,Data Product used as Input Data (DAT),Data Product used as Input Parameter X,Units of DAT,Units of X,DDATDX,MINDX,STARTDAT,TOLDAT Document Schema: array: origin: Array type: String instrument_class: origin: Instrument Class type: String reference_designator: origin: Reference Designator type: String dat: origin: Data Product used as Input Data (DAT) type: String x: origin: Data Product used as Input Parameter X type: String units_dat: origin: Units of DAT type: String units_x: origin: Units of X type: String d_dat_dx: origin: DDATDX type: float64 min_dx: origin: MINDX type: float64 start_dat: origin: STARTDAT type: float64 tol_dat: origin: TOLDAT type: float64 ''' sio = StringIO() sio.write(document) sio.seek(0) dr = DictReader(sio) for row in dr: try: key = '_'.join([ 'grad', row['Reference Designator'], row['Data Product used as Input Data (DAT)'], row['Data Product used as Input Parameter X'] ]) document = {} document['array'] = row['Array'] document['instrument_class'] = row['Instrument Class'] document['reference_designator'] = row['Reference Designator'] document['dat'] = row['Data Product used as Input Data (DAT)'] document['x'] = row['Data Product used as Input Parameter X'] document['units_dat'] = row['Units of DAT'] document['units_x'] = row['Units of X'] document['d_dat_dx'] = float(row['DDATDX']) document['min_dx'] = float(row['MINDX']) if row['MINDX'] else 0. document['start_dat'] = float( row['STARTDAT']) if row['STARTDAT'] else 0. document['tol_dat'] = float(row['TOLDAT']) yield key, document except TypeError: log.exception("Couldn't parse row") continue except ValueError: log.exception("Couldn't parse row") continue return