Example #1
0
    def _get_agent_instance_id(self, resource_id):
        dsaids, _ = self.rr.find_objects(subject=resource_id,
                                         predicate=PRED.hasAgentInstance,
                                         object_type=RT.ExternalDatasetAgentInstance,
                                         id_only=True)

        iaids, _ = self.rr.find_objects(subject=resource_id,
                                        predicate=PRED.hasAgentInstance,
                                        object_type=RT.InstrumentAgentInstance,
                                        id_only=True)

        paids, _ = self.rr.find_objects(subject=resource_id,
                                        predicate=PRED.hasAgentInstance,
                                        object_type=RT.PlatformAgentInstance,
                                        id_only=True)

        aids = dsaids + iaids + paids
        if len(aids) > 1:
            log.error("Multiple agent instances found")
            raise BadRequest("Failed to identify agent instance")

        if len(aids) == 0:
            log.error("Agent instance not found")
            raise BadRequest("Failed to identify agent instance")

        log.info("Found agent instance ID: %s", aids[0])
        return aids[0]
    def test_direct_access_vsp_client_disconnect(self):
        """
        Test agent direct_access mode for virtual serial port when shutdown by tcp client disconnect. 
        """

        self.assertInstrumentAgentState(ResourceAgentState.UNINITIALIZED)
    
        self.assertSetInstrumentState(ResourceAgentEvent.INITIALIZE, ResourceAgentState.INACTIVE)

        self.assertSetInstrumentState(ResourceAgentEvent.GO_ACTIVE, ResourceAgentState.IDLE)

        self.assertSetInstrumentState(ResourceAgentEvent.RUN, ResourceAgentState.COMMAND)

        cmd = AgentCommand(command=ResourceAgentEvent.GO_DIRECT_ACCESS,
                           kwargs={'session_type':DirectAccessTypes.vsp,
                           'session_timeout':600,
                           'inactivity_timeout':600})
        
        retval = self.assertSetInstrumentState(cmd, ResourceAgentState.DIRECT_ACCESS)

        log.info("GO_DIRECT_ACCESS retval=" + str(retval.result))

        tcp_client = self._start_tcp_client(retval)
        
        self.assertTrue(tcp_client.send_data('ts\r\n'))
        
        tcp_client.disconnect()

        self.assertInstrumentAgentState(ResourceAgentState.COMMAND, timeout=20)        
                
        self.assertSetInstrumentState(ResourceAgentEvent.RESET, ResourceAgentState.UNINITIALIZED)
        def start_DeviceStatusAlertEvent_subscriber(value_id, sub_type):
            """
            @return async_event_result  Use it to wait for the expected event
            """
            event_type = "DeviceStatusAlertEvent"

            async_event_result = AsyncResult()

            def consume_event(evt, *args, **kwargs):
                log.info('DeviceStatusAlertEvent_subscriber received evt: %s', str(evt))
                if evt.type_ != event_type or \
                   evt.value_id != value_id or \
                   evt.sub_type != sub_type:
                    return

                async_event_result.set(evt)

            kwargs = dict(event_type=event_type,
                          callback=consume_event,
                          origin=self.p_root.platform_device_id,
                          sub_type=sub_type)

            sub = EventSubscriber(**kwargs)
            sub.start()
            log.info("registered DeviceStatusAlertEvent subscriber: %s", kwargs)

            self._event_subscribers.append(sub)
            sub._ready_event.wait(timeout=self._receive_timeout)

            return async_event_result
Example #4
0
    def create_resources_snapshot(self, persist=False, filename=None):
        ds = CouchDataStore(DataStore.DS_RESOURCES, profile=DataStore.DS_PROFILE.RESOURCES, config=CFG, scope=self.sysname)
        all_objs = ds.find_docs_by_view("_all_docs", None, id_only=False)

        log.info("Found %s objects in datastore resources", len(all_objs))

        resources = {}
        associations = {}
        snapshot = dict(resources=resources, associations=associations)

        for obj_id, key, obj in all_objs:
            if obj_id.startswith("_design"):
                continue
            if not isinstance(obj, dict):
                raise Inconsistent("Object of bad type found: %s" % type(obj))
            obj_type = obj.get("type_", None)
            if obj_type == "Association":
                associations[obj_id] = obj.get("ts", None)
            elif obj_type:
                resources[obj_id] = obj.get("ts_updated", None)
            else:
                raise Inconsistent("Object with no type_ found: %s" % obj)

        if persist:
            dtstr = datetime.datetime.today().strftime('%Y%m%d_%H%M%S')
            path = filename or "interface/rrsnapshot_%s.json" % dtstr
            snapshot_json = json.dumps(snapshot)
            with open(path, "w") as f:
                #yaml.dump(snapshot, f, default_flow_style=False)
                f.write(snapshot_json)

        log.debug("Created resource registry snapshot. %s resources, %s associations", len(resources), len(associations))

        return snapshot
Example #5
0
    def init_driver_process_client(self):
        """
        @brief Launch the driver process and driver client
        @retval return driver process and driver client object
        """
        log.info("Startup Driver Process")

        this_pid = os.getpid()
        (dvr_proc, cmd_port, evt_port) = ZmqDriverProcess.launch_process(
            self._test_config.driver_module, self._test_config.driver_class, self._test_config.working_dir, this_pid
        )
        self.driver_process = dvr_proc
        log.info(
            "Started driver process for %d %d %s %s"
            % (cmd_port, evt_port, self._test_config.driver_module, self._test_config.driver_class)
        )
        log.info("Driver process pid %d" % self.driver_process.pid)

        # Create driver client.
        self.driver_client = ZmqDriverClient("localhost", cmd_port, evt_port)
        log.info(
            "Created driver client for %d %d %s %s"
            % (cmd_port, evt_port, self._test_config.driver_module, self._test_config.driver_class)
        )

        # Start client messaging.
        self.driver_client.start_messaging(self.event_received)
        log.info("Driver messaging started.")
        gevent.sleep(0.5)
Example #6
0
    def _call_plugins(self, method, process, config, **kwargs):
        bootstrap_plugins = config.get_safe("bootstrap_plugins", None)
        if bootstrap_plugins is None:
            log.warn("Bootstrapper called without bootstrap_plugins config")

        # Finding the system actor ID. If found, construct call context headers.
        # This may be called very early in bootstrap with no system actor yet existing
        system_actor, _ = process.container.resource_registry.find_resources(
            RT.ActorIdentity, name=self.CFG.system.system_actor, id_only=True
        )
        system_actor_id = system_actor[0] if system_actor else "anonymous"

        actor_headers = {
            "ion-actor-id": system_actor_id,
            "ion-actor-roles": {"ION": ["ION_MANAGER", "ORG_MANAGER"]} if system_actor else {},
        }

        # Set the call context of the current process
        with process.push_context(actor_headers):

            for plugin_info in bootstrap_plugins:
                plugin_mod, plugin_cls = plugin_info.get("plugin", [None, None])
                plugin_cfg = plugin_info.get("config", None)
                plugin_cfg = dict_merge(config, plugin_cfg) if plugin_cfg is not None else config

                try:
                    log.info("Bootstrapping plugin %s.%s ...", plugin_mod, plugin_cls)
                    plugin = for_name(plugin_mod, plugin_cls)
                    plugin_func = getattr(plugin, method)
                    plugin_func(process, plugin_cfg, **kwargs)
                except AbortBootstrap as abort:
                    raise
                except Exception as ex:
                    log.exception("Error bootstrapping plugin %s.%s", plugin_mod, plugin_cls)
    def _traverse(self, platform_id, parent_platform_objs=None):
        """
        Recursive routine that repeatedly calls _prepare_platform to build
        the object dictionary for each platform.

        @param platform_id ID of the platform to be visited
        @param parent_platform_objs dict of objects associated to parent
                        platform, if any.

        @retval the dict returned by _prepare_platform at this level.
        """

        log.info("Starting _traverse for %r", platform_id)

        plat_objs = self._prepare_platform(platform_id, parent_platform_objs)

        self.all_platforms[platform_id] = plat_objs

        # now, traverse the children:
        retval = self.rsn_oms.getSubplatformIDs(platform_id)
        subplatform_ids = retval[platform_id]
        for subplatform_id in subplatform_ids:
            self._traverse(subplatform_id, plat_objs)

        # note, topology indexed by platform_id
        self.topology[platform_id] = plat_objs['children']

        return plat_objs
Example #8
0
        def got_event(evt, *args, **kwargs):
            if not self._active:
                log.warn("%r: got_event called but manager has been destroyed",
                         self._platform_id)
                return

            if evt.type_ != event_type:
                log.trace("%r: ignoring event type %r. Only handle %r directly",
                          self._platform_id, evt.type_, event_type)
                return

            if evt.sub_type != sub_type:
                log.trace("%r: ignoring event sub_type %r. Only handle %r",
                          self._platform_id, evt.sub_type, sub_type)
                return

            state = self._agent.get_agent_state()

            statuses = formatted_statuses(self.aparam_aggstatus,
                                          self.aparam_child_agg_status,
                                          self.aparam_rollup_status)

            invalidated_children = self._agent._get_invalidated_children()

            log.info("%r/%s: (%s) status report triggered by diagnostic event:\n"
                     "%s\n"
                     "%40s : %s\n",
                     self._platform_id, state, self.resource_id, statuses,
                     "invalidated_children", invalidated_children)
    def persist_or_timeout(self, stream_id, rdt):
        """ retry writing coverage multiple times and eventually time out """
        done = False
        timeout = 2
        start = time.time()
        while not done:
            try:
                self.add_granule(stream_id, rdt)
                done = True
            except:
                log.exception('An issue with coverage, retrying after a bit')
                if (time.time() - start) > MAX_RETRY_TIME: # After an hour just give up
                    dataset_id = self.get_dataset(stream_id)
                    log.error("We're giving up, the coverage needs to be inspected %s", DatasetManagementService._get_coverage_path(dataset_id))
                    raise

                if stream_id in self._coverages:
                    log.info('Popping coverage for stream %s', stream_id)
                    self._coverages.pop(stream_id)

                gevent.sleep(timeout)
                if timeout > (60 * 5):
                    timeout = 60 * 5
                else:
                    timeout *= 2
Example #10
0
def get_user(username, password, *args, **kwargs):
    """ Callback to assert username/password and establish a user session.
    Used in password authentication flow. Called only during token creation.

    NOTE: If the same username logs in multiple times (e.g. different windows),
    then the most recent session overrides all others, which still remain valid
    """
    log.info("OAuth:get_user(%s)", username)
    if username and password:
        try:
            actor_id = ui_instance.idm_client.check_actor_credentials(username, password)
            actor_user = ui_instance.idm_client.read_actor_identity(actor_id)
            if actor_user.details.type_ != OT.UserIdentityDetails:
                log.warn("Bad identity details")
                return None

            # Create user session dict for ActorIdentity
            full_name = actor_user.details.contact.individual_names_given + " " + actor_user.details.contact.individual_name_family
            valid_until = int(get_ion_ts_millis() / 1000 + ui_instance.session_timeout)  # Int with seconds
            user_session = {"actor_id": actor_id, "user_id": actor_id, "username": username, "full_name": full_name, "valid_until": valid_until}
            actor_user.session = user_session

            ui_instance.container.resource_registry.update(actor_user)

            flask.g.actor_id = actor_id
            return user_session
        except NotFound:
            pass
    return None
    def test_25_get_set_params(self):
        self._initialize_and_run()

        p1 = TrhphParameter.TIME_BETWEEN_BURSTS
        result = self._get_params([p1])
        seconds = result[p1]
        new_seconds = seconds + 5
        if new_seconds > 30 or new_seconds < 15:
            new_seconds = 15

        p2 = TrhphParameter.VERBOSE_MODE
        new_verbose = self._get_verbose_flag_for_set_test()

        valid_params = {p1: new_seconds, p2: new_verbose}

        log.info("setting: %s" % str(valid_params))

        self._set_params(valid_params)

        result = self._get_params([p1, p2])

        seconds = result[p1]
        self.assertEqual(seconds, new_seconds)

        verbose = result[p2]
        self.assertEqual(verbose, new_verbose)
 def _stop_event_subscribers(self):
     """
     Stop event subscribers on cleanup.
     """
     log.info('cleanup: _stop_event_subscribers called.')
     for sub in self._event_subscribers:
         sub.stop()
    def check_authentication_token(self, token_string=''):
        """Checks given token and returns a dict with actor id if valid.

        @param token_string    str
        @retval token_info    dict
        @throws BadRequest    Illegal parameter type of value
        @throws NotFound    Token string not found
        @throws Unauthorized    Token not valid anymore or otherwise
        """
        token_id = "token_%s" % token_string
        token = self.container.object_store.read(token_id)
        if not isinstance(token, SecurityToken):
            raise Inconsistent("Token illegal type")
        if token.token_type != TokenTypeEnum.ACTOR_AUTH:
            raise BadRequest("Illegal token type")
        if token.token_string != token_string:
            raise Inconsistent("Found token's token_string does not match")
        cur_time = get_ion_ts_millis()
        if token.status != "OPEN":
            raise Unauthorized("Token status invalid")
        if cur_time >= int(token.expires):
            raise Unauthorized("Token expired")

        token_info = dict(actor_id=token.actor_id,
                    expiry=token.expires,
                    token=token,
                    token_id=token_id)

        log.info("Authentication token %s resolved to actor %s, expiry %s", token_string, token.actor_id, token.expires)

        return token_info
    def test_single_platform_with_an_instrument_and_deployments(self):
        #
        # basic test of launching a single platform with an instrument
        #
        self._set_receive_timeout()

        p_root = self._set_up_single_platform_with_some_instruments(['SBE37_SIM_01'])

        i_obj = self._setup_instruments['SBE37_SIM_01']

        platform_site_id, platform_deployment_id = self._create_platform_site_and_deployment(p_root.platform_device_id )

        instrument_site_id, instrument_deployment_id = self._create_instrument_site_and_deployment(platform_site_id=platform_site_id, instrument_device_id=i_obj.instrument_device_id)

        self._start_platform(p_root)
        self.addCleanup(self._stop_platform, p_root)
        self.addCleanup(self._run_shutdown_commands)

        self._run_startup_commands()

        ports = self._get_ports()
        log.info("_get_ports after startup= %s", ports)
        # the deployment should have turned port 1 on, but port 2 should still be off
        self.assertEquals(ports['1']['state'], 'ON')
        self.assertEquals(ports['2']['state'], None)
Example #15
0
    def on_restart(self, process, config, **kwargs):
        self.process = process
        inst_objs, _ = process.container.resource_registry.find_resources(restype=RT.Instrument, id_only=False)
        active_agents = []
        for inst in inst_objs:
            if len(inst.agent_state) >= 1:
                active_agents.append(inst._id)

        if not active_agents:
            return

        log.info("Restarting %s agents: %s", len(active_agents), active_agents)

        svc_client = ScionManagementProcessClient(process=process)
        for inst_id in active_agents:
            try:
                svc_client.start_agent(inst_id)
            except Exception as ex:
                log.exception("Cannot restart agent for %s" % inst_id)
                if "Agent already active" in ex.message:
                    try:
                        svc_client.stop_agent(inst_id)
                    except Exception:
                        pass
                    try:
                        svc_client.start_agent(inst_id)
                    except Exception:
                        log.warn("Agent stop/start for %s unsuccessful" % inst_id)
Example #16
0
def save_token(token, request, *args, **kwargs):
    """ Callback to persist newly created token. """
    log.info("OAuth:save_token(%s)", token)
    current_time = get_ion_ts_millis()
    # Note: token["expires_in"] was initialized by the framework and is not correct
    expires = str(current_time + 1000 * ui_instance.session_timeout)  # Str millis
    actor_id = flask.g.actor_id
    access_token_str = str(token["access_token"])
    refresh_token_str = str(token["refresh_token"])
    scopes = str(token["scope"])

    # Access token
    token_obj = SecurityToken(token_type=TokenTypeEnum.OAUTH_ACCESS, token_string=access_token_str,
                              expires=expires, status="OPEN", actor_id=actor_id,
                              attributes=dict(access_token=access_token_str, refresh_token=refresh_token_str,
                                              scopes=scopes, client_id=flask.g.client_id, ts_created=current_time))
    token_id = "access_token_%s" % access_token_str
    ui_instance.container.object_store.create(token_obj, token_id)

    # Refresh token
    token_obj = SecurityToken(token_type=TokenTypeEnum.OAUTH_REFRESH, token_string=refresh_token_str,
                              expires=expires, status="OPEN", actor_id=actor_id,
                              attributes=dict(access_token=access_token_str, refresh_token=refresh_token_str,
                                              scopes=scopes, client_id=flask.g.client_id, ts_created=current_time))
    token_id = "refresh_token_%s" % refresh_token_str
    ui_instance.container.object_store.create(token_obj, token_id)

    if ui_instance.remember_user:
        # Store user_id and access_token within server session, so that a client can resume
        flask.session["access_token"] = access_token_str
        flask.session["actor_id"] = actor_id
    def _trigger_func(self, stream_id):

        sine_ampl = 2.0 # Amplitude in both directions
        samples = 60
        sine_curr_deg = 0 # varies from 0 - 360

        startTime = time.time()
        count = samples #something other than zero

        while True:
            count = time.time() - startTime
            sine_curr_deg = (count % samples) * 360 / samples

            c = [sine_ampl * math.sin(math.radians(sine_curr_deg))]
            t = [sine_ampl * 2 * math.sin(math.radians(sine_curr_deg + 45))]
            p = [sine_ampl * 4 * math.sin(math.radians(sine_curr_deg + 60))]

            lat = lon = [0.0]
            tvar = [time.time()]

            ctd_packet = ctd_stream_packet(stream_id=stream_id,
                c=c, t=t, p = p, lat = lat, lon = lon, time=tvar)

            log.info('SinusoidalCtdPublisher sending 1 record!')
            self.publisher.publish(ctd_packet)

            time.sleep(1.0)
Example #18
0
def save_grant(client_id, code, request, *args, **kwargs):
    """ Saves a grant made by the user on provider to client """
    log.info("OAuth:save_grant(%s, %s)", client_id, code)
    # decide the expires time yourself
    expires = datetime.utcnow() + timedelta(seconds=100)
    grant = {}
    return grant
Example #19
0
    def delete_all_data(self, agent_instance_id, resource_id):
        # Delete Dataset and coverage for all original DataProducts
        self.delete_dataset(agent_instance_id, resource_id)

        res_obj = self.rr.read(resource_id)

        # Find parsed data product from device id
        count_dp, count_sd, count_st = 0, 0, 0
        dp_objs, _ = self.rr.find_subjects(RT.DataProduct, PRED.hasSource, resource_id, id_only=False)
        for dp_obj in dp_objs:
            # Find and delete Stream
            st_objs, _ = self.rr.find_objects(dp_obj._id, PRED.hasStream, RT.Stream, id_only=False)
            for st_obj in st_objs:
                self.rr.delete(st_obj._id)
                count_st += 1

            # Find and delete StreamDefinition
            sd_objs, _ = self.rr.find_objects(dp_obj._id, PRED.hasStreamDefinition, RT.StreamDefinition, id_only=False)
            for sd_obj in sd_objs:
                self.rr.delete(sd_obj._id)
                count_sd += 1

            # Delete DataProduct
            self.rr.delete(dp_obj._id)
            count_dp += 1

        log.info("Data resources deleted for device %s '%s': %s DataProduct, %s StreamDefinition, %s Stream",
                 resource_id, res_obj.name, count_dp, count_sd, count_st)
Example #20
0
    def start_service(self):
        """ Starts the web server. """
        if self.http_server is not None:
            self.stop_service()

        if self.server_socket_io:
            self.http_server = SocketIOServer((self.server_hostname, self.server_port),
                                              app.wsgi_app,
                                              resource='socket.io',
                                              log=None)
            self.http_server._gl = gevent.spawn(self.http_server.serve_forever)
            log.info("UI Server: Providing web sockets (socket.io) server")
        else:
            self.http_server = WSGIServer((self.server_hostname, self.server_port),
                                          app,
                                          log=None)
            self.http_server.start()

        if self.service_gateway:
            self.service_gateway.start()
            log.info("UI Server: Service Gateway started on %s", self.gateway_base_url)

        for ext_obj in self.extension_objs:
            ext_obj.on_start()

        return True
Example #21
0
    def delete_dataset(self, agent_instance_id, resource_id):

        res_obj = self.rr.read(resource_id)
        dpms = DataProductManagementServiceProcessClient(process=self)

        # Find data products from device id
        count_ds = 0
        dp_objs, _ = self.rr.find_objects(resource_id, PRED.hasOutputProduct, RT.DataProduct, id_only=False)
        for dp_obj in dp_objs:
            if dpms.is_persisted(dp_obj._id):
                raise BadRequest("DataProduct %s '%s' is currently persisted", dp_obj._id, dp_obj.name)

            ds_objs, _ = self.rr.find_objects(dp_obj._id, PRED.hasDataset, RT.Dataset, id_only=False)
            for ds_obj in ds_objs:
                # Delete coverage
                cov_path = DatasetManagementService._get_coverage_path(ds_obj._id)
                if os.path.exists(cov_path):
                    log.info("Removing coverage tree at %s", cov_path)
                    shutil.rmtree(cov_path)
                else:
                    raise OSError("Coverage path does not exist %s" % cov_path)

                # Delete Dataset and associations
                self.rr.delete(ds_obj._id)
                count_ds += 1

        log.info("Datasets and coverages deleted for device %s '%s': %s", resource_id, res_obj.name, count_ds)
 def process_packet_cb(packet, route, stream):
     if not isinstance(packet, DataPacket):
         log.warn("Received a non DataPacket message")
     self.recv_packets.append(packet)
     self.recv_rows += len(packet.data["data"])
     log.info("Received data packet #%s: rows=%s, cols=%s", len(self.recv_packets), len(packet.data["data"]),
              packet.data["cols"])
Example #23
0
    def recover_data(self, agent_instance_id, resource_id):
        res_obj = self.rr.read(resource_id)

        if res_obj.type_ != RT.InstrumentDevice:
            log.warn("Ignoring resource because it is not an instrument: %s - %s", res_obj.name, res_obj.type_)
            self._recover_data_status['ignored'].append("%s (%s)" % (res_obj.name, res_obj.type_))
            return

        self.recover_start = self.CFG.get("recover_start", None)
        self.recover_end = self.CFG.get("recover_end", None)

        if self.recover_end is None:
            raise BadRequest("Missing recover_end parameter")
        if self.recover_start is None:
            raise BadRequest("Missing recover_start parameter")

        try:
            ia_client = ResourceAgentClient(resource_id, process=self)
            log.info('Got ia client %s.', str(ia_client))
            ia_client.execute_resource(command=AgentCommand(command=DriverEvent.GAP_RECOVERY, args=[self.recover_start, self.recover_end]))

            self._recover_data_status['success'].append(res_obj.name)
        except Exception as e:
            log.warn("Failed to start recovery process for %s", res_obj.name)
            log.warn("Exception: %s", e)
            self._recover_data_status['fail'].append("%s (%s)" % (res_obj.name, e))
Example #24
0
    def init_port_agent(self):
        """
        @brief Launch the driver process and driver client.  This is used in the
        integration and qualification tests.  The port agent abstracts the physical
        interface with the instrument.
        @retval return the pid to the logger process
        """
        log.info("Startup Port Agent")
        # Create port agent object.
        this_pid = os.getpid()

        # Working dir and delim are hard coded here because this launch process
        # will change with the new port agent.
        self.port_agent = EthernetDeviceLogger.launch_process(
            self.comm_config.device_addr,
            self.comm_config.device_port,
            self._test_config.working_dir,
            self._test_config.delimeter,
            this_pid,
        )

        pid = self.port_agent.get_pid()
        while not pid:
            gevent.sleep(0.1)
            pid = self.port_agent.get_pid()

        port = self.port_agent.get_port()

        while not port:
            gevent.sleep(0.1)
            port = self.port_agent.get_port()

        log.info("Started port agent pid %d listening at port %d" % (pid, port))
        return port
Example #25
0
    def _set_calibration_for_data_product(self, dp_obj, dev_cfg):
        from ion.util.direct_coverage_utils import DirectCoverageAccess
        from coverage_model import SparseConstantType

        log.debug("Setting calibration for data product '%s'", dp_obj.name)
        dataset_ids, _ = self.rr.find_objects(dp_obj, PRED.hasDataset, id_only=True)
        publisher = EventPublisher(OT.InformationContentModifiedEvent)
        if not dataset_ids:
            data_product_management = DataProductManagementServiceProcessClient(process=self)
            log.debug(" Creating dataset for data product %s", dp_obj.name)
            data_product_management.create_dataset_for_data_product(dp_obj._id)
            dataset_ids, _ = self.rr.find_objects(dp_obj, PRED.hasDataset, id_only=True)
            if not dataset_ids:
                raise NotFound('No datasets were found for this data product, ensure that it was created')
        for dataset_id in dataset_ids:
            # Synchronize with ingestion
            with DirectCoverageAccess() as dca:
                cov = dca.get_editable_coverage(dataset_id)
                # Iterate over the calibrations
                for cal_name, contents in dev_cfg.iteritems():
                    if cal_name in cov.list_parameters() and isinstance(cov.get_parameter_context(cal_name).param_type, SparseConstantType):
                        value = float(contents['value'])
                        log.info(' Updating Calibrations for %s in %s', cal_name, dataset_id)
                        cov.set_parameter_values(cal_name, value)
                    else:
                        log.warn(" Calibration %s not found in dataset", cal_name)
                publisher.publish_event(origin=dataset_id, description="Calibrations Updated")
        publisher.close()
        log.info("Calibration set for data product '%s' in %s coverages", dp_obj.name, len(dataset_ids))
def start_fake_instrument_agent(container, stream_config={}, message_headers=None):

    # Create agent config.
    agent_config = {
        'driver_config' : DVR_CONFIG,
        'stream_config' : stream_config,
        'agent'         : {'resource_id': IA_RESOURCE_ID},
        'test_mode' : True
    }

    # Start instrument agent.

    log.debug("TestInstrumentAgent.setup(): starting IA.")
    container_client = ContainerAgentClient(node=container.node,
        name=container.name)

    ia_pid = container_client.spawn_process(name=IA_NAME,
        module=IA_MOD,
        cls=IA_CLS,
        config=agent_config, headers=message_headers)

    log.info('Agent pid=%s.', str(ia_pid))

    # Start a resource agent client to talk with the instrument agent.

    ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
    log.info('Got ia client %s.', str(ia_client))

    return ia_client
    def test_direct_access_telnet_session_setup_failure(self):
        """
        Test agent direct_access mode for telnet when session setup fails. 
        """

        self.assertInstrumentAgentState(ResourceAgentState.UNINITIALIZED)
    
        self.assertSetInstrumentState(ResourceAgentEvent.INITIALIZE, ResourceAgentState.INACTIVE)

        self.assertSetInstrumentState(ResourceAgentEvent.GO_ACTIVE, ResourceAgentState.IDLE)

        self.assertSetInstrumentState(ResourceAgentEvent.RUN, ResourceAgentState.COMMAND)

        cmd = AgentCommand(command=ResourceAgentEvent.GO_DIRECT_ACCESS,
                           kwargs={'session_type': DirectAccessTypes.telnet,
                           'session_timeout':600,
                           'inactivity_timeout':600})

        # test that DA session quits when bad token is sent
        retval = self.assertSetInstrumentState(cmd, ResourceAgentState.DIRECT_ACCESS)

        log.info("GO_DIRECT_ACCESS retval=" + str(retval.result))

        tcp_client = self._start_tcp_client(retval)
        
        self.assertTrue(tcp_client.start_telnet(token=retval.result['token'], handshake=False))

        self.assertInstrumentAgentState(ResourceAgentState.COMMAND, timeout=20)        
    def test_direct_access_vsp_inactivity_timeout(self):
        """
        Test agent direct_access mode for virtual serial port when shutdown by inactivity timeout. 
        """

        self.assertInstrumentAgentState(ResourceAgentState.UNINITIALIZED)
    
        self.assertSetInstrumentState(ResourceAgentEvent.INITIALIZE, ResourceAgentState.INACTIVE)

        self.assertSetInstrumentState(ResourceAgentEvent.GO_ACTIVE, ResourceAgentState.IDLE)

        self.assertSetInstrumentState(ResourceAgentEvent.RUN, ResourceAgentState.COMMAND)

        cmd = AgentCommand(command=ResourceAgentEvent.GO_DIRECT_ACCESS,
                           kwargs={'session_type':DirectAccessTypes.vsp,
                           'session_timeout':600,
                           'inactivity_timeout':10})
        
        retval = self.assertSetInstrumentState(cmd, ResourceAgentState.DIRECT_ACCESS)

        log.info("GO_DIRECT_ACCESS retval=" + str(retval.result))

        self._start_tcp_client(retval)

        self.assertInstrumentAgentState(ResourceAgentState.COMMAND, timeout=30)        
        
        # test for starting and inactivity timeout w/o ever connecting TCP client 
        retval = self.assertSetInstrumentState(cmd, ResourceAgentState.DIRECT_ACCESS)
        self.assertInstrumentAgentState(ResourceAgentState.COMMAND, timeout=30)        
        
        self.assertSetInstrumentState(ResourceAgentEvent.RESET, ResourceAgentState.UNINITIALIZED)
    def _set_resource(self):
        attrNames = self.ATTR_NAMES
        writ_attrNames = self.WRITABLE_ATTR_NAMES

        # do valid settings:

        # TODO more realistic value depending on attribute's type
        attrs = [(attrName, self.VALID_ATTR_VALUE) for attrName in attrNames]
        log.info("%r: setting attributes=%s", self.PLATFORM_ID, attrs)
        kwargs = dict(attrs=attrs)
        cmd = AgentCommand(command=PlatformAgentEvent.SET_RESOURCE, kwargs=kwargs)
        retval = self._execute_agent(cmd)
        attr_values = retval.result
        self.assertIsInstance(attr_values, dict)
        for attrName in attrNames:
            if attrName in writ_attrNames:
                self._verify_valid_attribute_id(attrName, attr_values)
            else:
                self._verify_not_writable_attribute_id(attrName, attr_values)

        # try invalid settings:

        # set invalid values to writable attributes:
        attrs = [(attrName, self.INVALID_ATTR_VALUE) for attrName in writ_attrNames]
        log.info("%r: setting attributes=%s", self.PLATFORM_ID, attrs)
        kwargs = dict(attrs=attrs)
        cmd = AgentCommand(command=PlatformAgentEvent.SET_RESOURCE, kwargs=kwargs)
        retval = self._execute_agent(cmd)
        attr_values = retval.result
        self.assertIsInstance(attr_values, dict)
        for attrName in writ_attrNames:
            self._verify_attribute_value_out_of_range(attrName, attr_values)
 def consume_event(*args, **kwargs):
     log.info('Test recieved ION event: args=%s, kwargs=%s, event=%s.', 
              str(args), str(kwargs), str(args[0]))
     self._events_received.append(args[0])
     if self._event_count > 0 and \
         self._event_count == len(self._events_received):
         self._async_event_result.set()
 def on_start(self):
     op = self.CFG.get("op", None)
     log.info("LoadSystemPolicy: {op=%s}" % op)
     if op:
         if op == "load":
             self.op_load_system_policies(self)
         else:
             raise iex.BadRequest("Operation unknown")
     else:
         raise iex.BadRequest("No operation specified")
Example #32
0
    def test(self):

        self._configure()
        self._connect()
        self._ping()

        self._get_attribute_values()

        log.info("sleeping to eventually see some events...")
        sleep(15)
Example #33
0
    def setUp(self):
        self._start_container()
        self.container.start_rel_from_url('res/deploy/r2cei.yml')
        #self.pd_cli = ProcessDispatcherServiceClient(node=self.container.node)
        self.pd_cli = ProcessDispatcherServiceClient(to_name="process_dispatcher")

        self.process_definition_id = uuid4().hex
        self.process_definition_name = 'test'
        self.process_definition =  ProcessDefinition(name=self.process_definition_name, executable={
                'module': 'ion.agents.cei.test.test_haagent',
                'class': 'TestProcess'
        })
        self.pd_cli.create_process_definition(self.process_definition, self.process_definition_id)

        self.resource_id = "haagent_1234"
        self._haa_name = "high_availability_agent"
        self._haa_dashi_name = "dashi_haa_" + uuid4().hex
        self._haa_dashi_uri = get_dashi_uri_from_cfg()
        self._haa_dashi_exchange = "%s.hatests" % bootstrap.get_sys_name()
        self._haa_config = {
            'highavailability': {
                'policy': {
                    'interval': 1,
                    'name': 'npreserving',
                    'parameters': {
                        'preserve_n': 0
                    }
                },
                'process_definition_id': self.process_definition_id,
                'dashi_messaging' : True,
                'dashi_exchange' : self._haa_dashi_exchange,
                'dashi_name': self._haa_dashi_name
            },
            'agent': {'resource_id': self.resource_id},
        }

        self._base_services, _ = self.container.resource_registry.find_resources(
                restype="Service", name=self.process_definition_name)

        self._base_procs = self.pd_cli.list_processes()

        self.waiter = ProcessStateWaiter()
        self.waiter.start()

        self.container_client = ContainerAgentClient(node=self.container.node,
            name=self.container.name)
        self._haa_pid = self.container_client.spawn_process(name=self._haa_name,
            module="ion.agents.cei.high_availability_agent",
            cls="HighAvailabilityAgent", config=self._haa_config)

        # Start a resource agent client to talk with the instrument agent.
        self._haa_pyon_client = SimpleResourceAgentClient(self.resource_id, process=FakeProcess())
        log.info('Got haa client %s.', str(self._haa_pyon_client))

        self.haa_client = HighAvailabilityAgentClient(self._haa_pyon_client)
Example #34
0
    def setUp(self):
        self._start_container()
        self.container.start_rel_from_url('res/deploy/r2cei.yml')

        self.resource_id = "eeagent_1234"
        self._eea_name = "eeagent"

        self.supd_directory = tempfile.mkdtemp()

        self.agent_config = {
            'eeagent': {
                'heartbeat': 0,
                'slots': 100,
                'name': 'pyon_eeagent',
                'launch_type': {
                    'name': 'supd',
                    'pyon_directory': os.getcwd(),
                    'supd_directory': self.supd_directory,
                    'supdexe': 'bin/supervisord'
                },
            },
            'agent': {
                'resource_id': self.resource_id
            },
            'logging': {
                'loggers': {
                    'eeagent': {
                        'level': 'DEBUG',
                        'handlers': ['console']
                    }
                },
                'root': {
                    'handlers': ['console']
                },
            }
        }

        # Start eeagent.
        self._eea_pid = None

        self.container_client = ContainerAgentClient(node=self.container.node,
                                                     name=self.container.name)
        self._eea_pid = self.container_client.spawn_process(
            name=self._eea_name,
            module="ion.agents.cei.execution_engine_agent",
            cls="ExecutionEngineAgent",
            config=self.agent_config)
        log.info('Agent pid=%s.', str(self._eea_pid))

        # Start a resource agent client to talk with the instrument agent.
        self._eea_pyon_client = SimpleResourceAgentClient(
            self.resource_id, process=FakeProcess())
        log.info('Got eea client %s.', str(self._eea_pyon_client))

        self.eea_client = ExecutionEngineAgentClient(self._eea_pyon_client)
Example #35
0
    def _load_DataProductLink(self, row):
        log.info("Loading DataProductLink")

        dp_id = self.resource_ids[row["data_product_id"]]
        res_id = self.resource_ids[row["input_resource_id"]]

        create_stream = self._get_typed_value(row["create_stream"],
                                              targettype="bool")

        svc_client = self._get_service_client("data_acquisition_management")
        svc_client.assign_data_product(res_id, dp_id, create_stream)
    def _get_attribute_values(self):
        attrNames = self.ATTR_NAMES

        # see OOIION-631 note in test_platform_agent_with_rsn
        from_time = str(int(get_ion_ts()) - 50000)  # a 50-sec time window
        req_attrs = [(attr_id, from_time) for attr_id in attrNames]
        attr_values = self._plat_driver.get_attribute_values(req_attrs)
        log.info("attr_values = %s" % str(attr_values))
        self.assertIsInstance(attr_values, dict)
        for attr_name in attrNames:
            self.assertTrue(attr_name in attr_values)
Example #37
0
 def _connect_instrument(self):
     platform_id = self.PLATFORM_ID
     port_id = self.PORT_ID
     instrument_id = self.INSTRUMENT_ID
     attributes = {'maxCurrentDraw': 1, 'initCurrent': 2,
                   'dataThroughput': 3, 'instrumentType': 'FOO'}
     retval = self._rsn_oms.instr.connect_instrument(platform_id, port_id,
                                                     instrument_id, attributes)
     log.info("connect_instrument = %s" % retval)
     ports = self._verify_valid_platform_id(platform_id, retval)
     self._verify_valid_port_id(port_id, ports)
Example #38
0
    def _prune_dataset(self, data_file):
        if not self.prune_mode:
            return
        if self.prune_mode == "max_age_rel":
            # Prunes if first timestamp older than trigger compared to most recent timestamp
            trigger_age = float(self.pruning_attrs.get("trigger_age", 0))
            retain_age = float(self.pruning_attrs.get("retain_age", 0))
            if trigger_age <= 0.0 or retain_age <= 0.0 or trigger_age < retain_age:
                raise BadRequest("Bad pruning trigger_age or retain_age")
            var_ds = data_file["vars/%s" % self.time_var]
            cur_idx = var_ds.attrs["cur_row"]
            if not len(var_ds) or not cur_idx:
                return
            min_ts = NTP4Time.from_ntp64(var_ds[0].tostring()).to_unix()
            max_ts = NTP4Time.from_ntp64(var_ds[cur_idx -
                                                1].tostring()).to_unix()
            if min_ts + trigger_age >= max_ts:
                return

            # Find the first index that is lower or equal to retain_age and delete gap
            start_time = (max_ts - retain_age) * 1000
            log.info("PRUNING dataset now: mode=%s, start_time=%s",
                     self.prune_mode, int(start_time))
            copy_filename = self._get_data_copy(
                data_file, data_filter=dict(start_time=start_time))
        elif self.prune_mode == "max_age_abs":
            # Prunes if first timestamp older than trigger compared to current timestamp
            raise NotImplementedError()
        elif self.prune_mode == "max_rows":
            raise NotImplementedError()
        elif self.prune_mode == "max_size":
            raise NotImplementedError()
        else:
            raise BadRequest("Invalid prune_mode: %s" % self.prune_mode)

        if not copy_filename:
            return

        # Do the replace of data file with the copy.
        # Make sure to heed race conditions so that waiting processes won't lock the file first
        ds_filename = self._get_ds_filename()
        ds_filename_bak = ds_filename + ".bak"
        if os.path.exists(ds_filename_bak):
            os.remove(ds_filename_bak)

        data_file.close(
        )  # Note: Inter-process race condition possible because close removes the lock
        shutil.move(ds_filename, ds_filename_bak)
        shutil.move(
            copy_filename,
            ds_filename)  # Note: There may be a cross-device-link error here
        # os.remove(ds_filename_bak)
        log.info("Pruning successful. Replaced dataset with pruned file.")
        return True
Example #39
0
    def recv_packet(self, packet, in_stream_route, in_stream_id):
        #Check to see if the class instance was set up as a event triggered transform. If yes, skip the packet
        if self.event_timer_interval:
            return

        log.info('Received packet')
        mpl_data_granule = VizTransformMatplotlibGraphsAlgorithm.execute(
            packet, params=self.get_stream_definition())
        for stream_name in self.stream_names:
            publisher = getattr(self, stream_name)
            publisher.publish(mpl_data_granule)
 def _check_lock(self):
     LOCK_TIMEOUT = 0.5
     cur_time = get_ion_ts_millis()
     has_lock = self._is_master()
     self.has_lock = self.container.directory.acquire_lock(
         "coord", LOCK_TIMEOUT, self.id)
     if self.has_lock:
         # We are the master
         if not has_lock:
             log.info("Process %s is now the master", self._proc_name)
         self.lock_expires = cur_time + int(1000 * LOCK_TIMEOUT)
Example #41
0
 def stop_port_agent(self):
     """
     Stop the port agent.
     """
     if self.port_agent:
         pid = self.port_agent.get_pid()
         if pid:
             log.info('Stopping pagent pid %i' % pid)
             self.port_agent.stop()
         else:
             log.info('No port agent running.')
    def _turn_off_port(self, port_id=None):
        src = self.__class__.__name__
        port_id = port_id or self.PORT_ID
        kwargs = dict(port_id=port_id, src=src)

        result = self._execute_resource(RSNPlatformDriverEvent.TURN_OFF_PORT,
                                        **kwargs)
        log.info("TURN_OFF_PORT = %s", result)
        self.assertIsInstance(result, dict)
        self.assertTrue(port_id in result)
        self.assertEquals(result[port_id], NormalResponse.PORT_TURNED_OFF)
 def validate_granule_subscription(self, msg, route, stream_id):
     '''
     Validation for granule format
     '''
     if msg == {}:
         return
     rdt = RecordDictionaryTool.load_from_granule(msg)
     log.info('%s', rdt.pretty_print())
     self.assertIsInstance(
         msg, Granule, 'Message is improperly formatted. (%s)' % type(msg))
     self.event.set()
Example #44
0
    def rcmd_makesay(self, who, what, *args, **kwargs):
        log.info("Makesay: Relaying command %s to %s" % (what, who))
        target_name = agent_instances.get(str(who), None)
        if target_name:
            self.rac = ResourceAgentClient(resource_id='R', name=target_name, process=self)

            cmd = AgentCommand(command='say', args=[what])
            res = self.rac.execute(cmd)
            return "OK"
        else:
            return "UNKNOWN AGENT"
Example #45
0
    def on_init(self):
        # Time in between event persists
        self.persist_interval = float(
            self.CFG.get_safe("process.event_persister.persist_interval", 1.0))

        self.persist_blacklist = self.CFG.get_safe(
            "process.event_persister.persist_blacklist", {})

        self._event_type_blacklist = [
            entry['event_type'] for entry in self.persist_blacklist
            if entry.get('event_type', None) and len(entry) == 1
        ]
        self._complex_blacklist = [
            entry for entry in self.persist_blacklist
            if not (entry.get('event_type', None) and len(entry) == 1)
        ]
        if self._complex_blacklist:
            log.warn(
                "EventPersister does not yet support complex blacklist expressions: %s",
                self._complex_blacklist)

        # Holds received events FIFO in synchronized queue
        self.event_queue = Queue()

        # Temporarily holds list of events to persist while datastore operation are not yet completed
        # This is where events to persist will remain if datastore operation fails occasionally.
        self.events_to_persist = None

        # Number of unsuccessful consecutive attempts to persist during loop
        self.failure_count = 0

        # bookkeeping for greenlet
        self._persist_greenlet = None
        self._terminate_persist = Event(
        )  # when set, exits the persister greenlet

        # The event subscriber
        self.event_sub = None

        process_plugin_defs = self.CFG.get_safe(
            "process.event_persister.process_plugins", {}) or {}

        # Registered event process plugins
        self.process_plugins = {}
        for plugin_name, plugin_cls, plugin_args in process_plugin_defs:
            try:
                plugin = named_any(plugin_cls)(**plugin_args)
                self.process_plugins[plugin_name] = plugin
                log.info("Loaded event processing plugin %s (%s)", plugin_name,
                         plugin_cls)
            except Exception as ex:
                log.error(
                    "Cannot instantiate event processing plugin %s (%s): %s",
                    plugin_name, plugin_cls, ex)
Example #46
0
    def _get_attribute_values(self):
        attrNames = self.ATTR_NAMES

        # see OOIION-631 note in test_platform_agent
        from_time = str(int(get_ion_ts()) - 50000)  # a 50-sec time window
        req_attrs = [(attr_id, from_time) for attr_id in attrNames]
        result = self._on_event(RSNPlatformDriverEvent.GET, attrs=req_attrs)
        log.info("_get_attribute_values result = %s", result)
        self.assertIsInstance(result, dict)
        for attr_name in attrNames:
            self.assertTrue(attr_name in result)
 def invalidate_authentication_token(self, token_string=''):
     """Invalidates an authentication token, but leaves it in place for auditing purposes.
     """
     token_id = "token_%s" % token_string
     token = self.container.object_store.read(token_id)
     if not isinstance(token, SecurityToken):
         raise Inconsistent("Token illegal type")
     if token.token_type != TokenTypeEnum.ACTOR_AUTH:
         raise BadRequest("Illegal token type")
     token.status = "INVALID"
     self.container.object_store.update(token)
     log.info("Invalidated security auth token: %s", token.token_string)
Example #48
0
def find_org_roles(requester=''):

    find_org_request = {
        "serviceRequest": {
            "serviceName": "org_management",
            "serviceOp": "find_org",
            "expiry": 0,
            "params": {}
        }
    }

    if requester is not None:
        find_org_request["serviceRequest"]["requester"] = requester

    response = gateway_request('org_management/find_org',
                               simplejson.dumps(find_org_request))

    if response['data'].has_key(GATEWAY_ERROR):
        log.error(response['data'][GATEWAY_ERROR][GATEWAY_ERROR_MESSAGE])
        return response['data'][GATEWAY_ERROR][GATEWAY_ERROR_MESSAGE]

    response_data = response['data'][GATEWAY_RESPONSE]
    ion_org_id = response_data['_id']

    find_org_roles_request = {
        "serviceRequest": {
            "serviceName": "org_management",
            "serviceOp": "find_org_roles",
            "expiry": 0,
            "params": {
                "org_id": ion_org_id
            }
        }
    }

    if requester is not None:
        find_org_roles_request["serviceRequest"]["requester"] = requester

    response = gateway_request('org_management/find_org_roles',
                               simplejson.dumps(find_org_roles_request))

    if response['data'].has_key(GATEWAY_ERROR):
        log.error(response['data'][GATEWAY_ERROR][GATEWAY_ERROR_MESSAGE])
        return response['data'][GATEWAY_ERROR][GATEWAY_ERROR_MESSAGE]

    response_data = response['data'][GATEWAY_RESPONSE]

    log.info('Number of UserRole objects in the ION Org: %s' %
             (str(len(response_data))))
    for res in response_data:
        log.debug(res)

    return response_data
Example #49
0
    def _load_PlatformModel_OOI(self):
        log.info("Loading OOI PlatformModel assets")
        for pm_def in self.platform_models.values():
            fakerow = {}
            fakerow[self.COL_ID] = pm_def['code']
            fakerow['pm/name'] = "%s (%s)" % (pm_def['code'], pm_def['name'])
            fakerow['pm/description'] = pm_def['name']
            fakerow['pm/OOI_node_type'] = pm_def['code']
            mf_id = 'MF_RSN' if pm_def['code'].startswith("R") else 'MF_CGSN'
            fakerow['mf_ids'] = mf_id

            self._load_PlatformModel(fakerow)
Example #50
0
    def persist_events_state(self, event_list):
        # Get events for each device
        events_by_dev = {}
        for evt in event_list:
            if evt.type_ not in RECOGNIZED_EVENTS:
                continue
            origin = evt.origin
            if origin:
                events_by_dev.setdefault(origin, []).append(evt)

        if not events_by_dev:
            return

        # Get existing states
        dev_ids = events_by_dev.keys()
        dev_states = self.read_states(dev_ids)
        states_by_dev = dict(zip(dev_ids, dev_states))
        existing_states = [
            did for did, ds in states_by_dev.iteritems() if ds is not None
        ]
        log.debug("Found %s persisted device states", len(existing_states))

        # Update state using event
        for dev_id, dev_evts in events_by_dev.iteritems():
            dev_state = states_by_dev.get(dev_id, None)
            if dev_state is None:
                dev_state = self.create_device_state(STATE_PREFIX + dev_id,
                                                     dev_id)
                states_by_dev[dev_id] = dev_state
            for evt in dev_evts:
                self.update_device_state(dev_state, dev_id, evt)

        # Persist events
        upd_dev_states = [
            ds for did, ds in states_by_dev.iteritems()
            if did in existing_states
        ]
        self.store.update_doc_mult(upd_dev_states)

        new_dev_states = [
            ds for did, ds in states_by_dev.iteritems()
            if did not in existing_states
        ]
        new_dev_state_ids = [
            STATE_PREFIX + ds["device_id"] for ds in new_dev_states
        ]
        self.store.create_doc_mult(new_dev_states,
                                   object_ids=new_dev_state_ids)

        log.info(
            "Processed and persisted %s status events for %s devices (%s upd/%s new)",
            len(event_list), len(dev_ids), len(upd_dev_states),
            len(new_dev_states))
Example #51
0
    def _load_InstrumentModel_OOI(self):
        log.info("Loading OOI InstrumentModel assets")
        for im_def in self.instrument_models.values():
            fakerow = {}
            fakerow[self.COL_ID] = im_def['code']
            fakerow['im/name'] = im_def['name']
            fakerow['im/description'] = im_def['name']
            fakerow['im/instrument_family'] = im_def['family']
            fakerow['im/instrument_class'] = im_def['code']
            fakerow['mf_ids'] = 'MF_RSN,MF_CGSN'

            self._load_InstrumentModel(fakerow)
    def parse(self, search_request='', id_only=True, search_args=None):
        """Parses a given string request and assembles the query, processes the query and returns the results of the query.
        See the query language definition: https://confluence.oceanobservatories.org/display/CIDev/Discovery+Service+Query+Format

        @param search_request    str
        @param id_only    bool
        @param search_args dict
        @retval results    list
        """
        log.info("Search DSL: %s", search_request)
        query_request = self._parse_query_string(search_request)
        return self._discovery_request(query_request, id_only=id_only, search_args=search_args, query_params=search_args)
    def _trigger_func(self, stream_id):

        while True:

            length = random.randint(1, 20)

            ctd_packet = self._get_ctd_packet(stream_id, length)

            log.info('SimpleCtdPublisher sending %d values!' % length)
            self.publisher.publish(ctd_packet)

            time.sleep(2.0)
Example #54
0
 def _check_lock(self):
     cur_time = get_ion_ts_millis()
     was_leader = self.is_leader()
     self._has_lock = self.container.directory.acquire_lock(
         self.scope, self._lock_timeout, self.instance_id)
     if self._has_lock:
         # We are the leader
         if not was_leader:
             log.info("Process %s is now the leader for '%s'",
                      self.instance_id, self.scope)
             self._inform_acquire()
         self._lock_expires = cur_time + int(1000 * self._lock_timeout)
Example #55
0
    def _persister_loop(self, persist_interval):
        log.debug('Starting event persister thread with persist_interval=%s',
                  persist_interval)

        # Event.wait returns False on timeout (and True when set in on_quit), so we use this to both exit cleanly and do our timeout in a loop
        while not self._terminate_persist.wait(timeout=persist_interval):
            try:
                if self.events_to_persist and self.failure_count > 2:
                    bad_events = []
                    log.warn("Attempting to persist %s events individually" %
                             (len(self.events_to_persist)))
                    for event in self.events_to_persist:
                        try:
                            self.container.event_repository.put_event(event)
                        except Exception:
                            bad_events.append(event)

                    if len(self.events_to_persist) != len(bad_events):
                        log.warn(
                            "Succeeded to persist some of the events - rest must be bad"
                        )
                        self._log_events(bad_events)
                    elif bad_events:
                        log.error("Discarding %s events after %s attempts!!" %
                                  (len(bad_events), self.failure_count))
                        self._log_events(bad_events)

                    self.events_to_persist = None
                    self.failure_count = 0

                elif self.events_to_persist:
                    # There was an error last time and we need to retry
                    log.info("Retry persisting %s events" %
                             len(self.events_to_persist))
                    self._persist_events(self.events_to_persist)
                    self.events_to_persist = None

                self.events_to_persist = [
                    self.event_queue.get()
                    for x in xrange(self.event_queue.qsize())
                ]

                self._persist_events(self.events_to_persist)
                self.events_to_persist = None
                self.failure_count = 0
            except Exception as ex:
                # Note: Persisting events may fail occasionally during test runs (when the "events" datastore is force
                # deleted and recreated). We'll log and keep retrying forever.
                log.exception(
                    "Failed to persist %s received events. Will retry next cycle"
                    % len(self.events_to_persist))
                self.failure_count += 1
                self._log_events(self.events_to_persist)
 def _reset(self):
     """
     Set as a clean-up to make sure the agent is reset after each test,
     so the driver is stopped.
     """
     log.info("_reset called: sending 'reset' command to agent")
     cmd = AgentCommand(command='reset')
     retval = self._ia_client.execute_agent(cmd)
     cmd = AgentCommand(command='get_agent_state')
     retval = self._ia_client.execute_agent(cmd)
     state = retval.result
     self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
Example #57
0
 def on_start(self):
     op = self.CFG.get("op", None)
     log.info("DirectoryAdmin: {op=%s}" % (op))
     if op:
         if op == "register":
             self.op_register_definitions()
         elif op == "load":
             pass
         else:
             raise iex.BadRequest("Operation unknown")
     else:
         raise iex.BadRequest("No operation specified")
Example #58
0
    def _turn_off_port(self):
        # TODO real settings and corresp verification

        port_id = self.PORT_ID

        kwargs = dict(port_id=port_id)
        result = self._execute_resource(RSNPlatformDriverEvent.TURN_OFF_PORT,
                                        **kwargs)
        log.info("TURN_OFF_PORT = %s", result)
        self.assertIsInstance(result, dict)
        self.assertTrue(port_id in result)
        self.assertEquals(result[port_id], NormalResponse.PORT_TURNED_OFF)
    def test_80_execute_diagnostics(self):
        self._initialize_and_run()

        log.info("executing diagnostics")
        num_scans = 11
        cmd = AgentCommand(command='diagnostics',
                           kwargs=dict(num_scans=num_scans,
                                       timeout=self._timeout))
        reply = self._ia_client.execute(cmd)
        log.info("diagnostics reply = %s" % str(reply))
        self.assertTrue(isinstance(reply.result, list))
        self.assertEqual(len(reply.result), num_scans)
Example #60
0
 def stop_streaming(self):
     if self.current_state == self.AGENTSTATE_CONNECTED:
         return
     elif self.current_state != self.AGENTSTATE_STREAMING:
         raise BadRequest("Illegal agent state: %s" % self.current_state)
     log.info("Stop streaming")
     try:
         res = self.on_stop_streaming()
     except Exception:
         self.current_state = self.AGENTSTATE_ERROR
         raise
     self.current_state = self.AGENTSTATE_CONNECTED