Beispiel #1
0
    def parse(cls, raw):
        """
        Given a possibly-multiline string representing an SSE message, parse it
        and return a Event object.
        """
        msg = cls()
        for line in raw.split('\n'):
            m = cls.sse_line_pattern.match(line)
            if m is None:
                # Malformed line.  Discard but warn.
                log.warning('Invalid SSE line: "%s"' % line)
                continue

            name = m.groupdict()['name']
            value = m.groupdict()['value']
            if name == '':
                # line began with a ":", so is a comment.  Ignore
                continue

            if name == 'data':
                # If we already have some data, then join to it with a newline.
                # Else this is it.
                if msg.data:
                    msg.data = '%s\n%s' % (msg.data, value)
                else:
                    msg.data = value
            elif name == 'event':
                msg.event = value
            elif name == 'id':
                msg.id = value
            elif name == 'retry':
                msg.retry = int(value)

        return msg
Beispiel #2
0
 def run(self):
     log.info("Starting StreamingUpdateProcessor connecting to uri: " + self._uri)
     self._running = True
     while self._running:
         try:
             messages = self._connect()
             for msg in messages:
                 if not self._running:
                     break
                 message_ok = self.process_message(self._store, self._requester, msg)
                 if message_ok is True and self._ready.is_set() is False:
                     log.info("StreamingUpdateProcessor initialized ok.")
                     self._ready.set()
         except HTTPError as e:
             log.error("Received unexpected status code %d for stream connection" % e.response.status_code)
             if e.response.status_code == 401:
                 log.error("Received 401 error, no further streaming connection will be made since SDK key is invalid")
                 self.stop()
                 break
             else:
                 log.warning("Restarting stream connection after one second.")
         except Exception:
             log.warning("Caught exception. Restarting stream connection after one second.",
                         exc_info=True)
         time.sleep(1)
    def _make_update_processor(self, config, store, ready,
                               diagnostic_accumulator):
        if config.update_processor_class:
            log.info("Using user-specified update processor: " +
                     str(config.update_processor_class))
            return config.update_processor_class(config, store, ready)

        if config.offline or config.use_ldd:
            return NullUpdateProcessor(config, store, ready)

        if config.feature_requester_class:
            feature_requester = config.feature_requester_class(config)
        else:
            feature_requester = FeatureRequesterImpl(config)
        """ :type: FeatureRequester """

        if config.stream:
            return StreamingUpdateProcessor(config, feature_requester, store,
                                            ready, diagnostic_accumulator)

        log.info("Disabling streaming API")
        log.warning(
            "You should only disable the streaming API if instructed to do so by LaunchDarkly support"
        )
        return PollingUpdateProcessor(config, feature_requester, store, ready)
        def do_send(should_retry):
            # noinspection PyBroadException
            try:
                if isinstance(events, dict):
                    body = [events]
                else:
                    body = events

                json_body = jsonpickle.encode(body, unpicklable=False)
                log.debug('Sending events payload: ' + json_body)
                hdrs = _headers(self.sdk_key)
                uri = self._config.events_uri
                r = self._session.post(uri,
                                       headers=hdrs,
                                       timeout=(self._config.connect_timeout,
                                                self._config.read_timeout),
                                       data=json_body)
                r.raise_for_status()
            except ProtocolError as e:
                if e.args is not None and len(
                        e.args) > 1 and e.args[1] is not None:
                    inner = e.args[1]
                    if inner.errno is not None and inner.errno == errno.ECONNRESET and should_retry:
                        log.warning(
                            'ProtocolError exception caught while sending events. Retrying.'
                        )
                        do_send(False)
                else:
                    log.exception(
                        'Unhandled exception in event consumer. Analytics events were not processed.'
                    )
            except:
                log.exception(
                    'Unhandled exception in event consumer. Analytics events were not processed.'
                )
Beispiel #5
0
 def process_message(store, requester, msg, ready):
     payload = json.loads(msg.data)
     log.debug("Received stream event {}".format(msg.event))
     if msg.event == 'put':
         store.init(payload)
         if not ready.is_set() and store.initialized:
             ready.set()
             log.info("StreamingUpdateProcessor initialized ok")
     elif msg.event == 'patch':
         key = payload['path'][1:]
         feature = payload['data']
         log.debug("Updating feature {}".format(key))
         store.upsert(key, feature)
     elif msg.event == "indirect/patch":
         key = payload['data']
         store.upsert(key, requester.get_one(key))
     elif msg.event == "indirect/put":
         store.init(requester.get_all())
         if not ready.is_set() and store.initialized:
             ready.set()
             log.info("StreamingUpdateProcessor initialized ok")
     elif msg.event == 'delete':
         key = payload['path'][1:]
         # noinspection PyShadowingNames
         version = payload['version']
         store.delete(key, version)
     else:
         log.warning('Unhandled event in stream processor: ' + msg.event)
 def toggle(self, key, default):
     """Deprecated synonym for :func:`variation()`.
     .. deprecated:: 2.0.0
     """
     log.warning(
         "Deprecated method: toggle() called. Use variation() instead.")
     return self.variation(key, default)
Beispiel #7
0
 def run(self):
     log.info("Starting MobileStreamingUpdateProcessor connecting to uri: " + self._uri)
     self._running = True
     while self._running:
         try:
             init_start = time.time()
             messages = self._connect()
             for msg in messages:
                 if not self._running:
                     break
                 message_ok = self.process_message(self._store, self._requester, msg)
                 if message_ok is True and self._ready.is_set() is False:
                     log.info("MobileStreamingUpdateProcessor initialized ok.")
                     init_duration = int((time.time() - init_start) * 1000)
                     request_success.fire(request_type="ld:init", name="mobile", response_time=init_duration, response_length=0)
                     self._ready.set()
         except UnsuccessfulResponseException as e:
             log.error(http_error_message(e.status, "stream connection"))
             init_duration = int((time.time() - init_start) * 1000)
             request_failure.fire(request_type="ld:init", name="mobile", response_time=init_duration, response_length=0, exception=e)
             if not is_http_error_recoverable(e.status):
                 self._ready.set()  # if client is initializing, make it stop waiting; has no effect if already inited
                 self.stop()
                 break
         except Exception as e:
             init_duration = int((time.time() - init_start) * 1000)
             request_failure.fire(request_type="ld:init", name="mobile", response_time=init_duration, response_length=0, exception=e)
             log.warning("Caught exception. Restarting stream connection after one second. %s" % e)
             # no stacktrace here because, for a typical connection error, it'll just be a lengthy tour of urllib3 internals
         time.sleep(1)
Beispiel #8
0
 def do_send(should_retry):
     # noinspection PyBroadException
     try:
         if isinstance(events, dict):
             body = [events]
         else:
             body = events
         hdrs = _headers(self._sdk_key)
         r = yield self._session.post(
             self._config.events_uri,
             headers=hdrs,
             timeout=(self._config.connect_timeout,
                      self._config.read_timeout),
             data=json.dumps(body))
         r.raise_for_status()
     except ProtocolError as e:
         inner = e.args[1]
         if inner.errno == errno.ECONNRESET and should_retry:
             log.warning(
                 'ProtocolError exception caught while sending events. Retrying.'
             )
             yield do_send(False)
         else:
             log.exception(
                 'Unhandled exception in event consumer. Analytics events were not processed.'
             )
     except:
         log.exception(
             'Unhandled exception in event consumer. Analytics events were not processed.'
         )
Beispiel #9
0
 def run(self):
     try:
         output_events = self._formatter.make_output_events(self._payload.events, self._payload.summary)
         resp = self._do_send(output_events)
     except Exception:
         log.warning(
             'Unhandled exception in event processor. Analytics events were not processed.',
             exc_info=True)
Beispiel #10
0
 def add_event(self, event):
     if len(self._events) >= self._capacity:
         if not self._exceeded_capacity:
             log.warning("Event queue is full-- dropped an event")
             self._exceeded_capacity = True
     else:
         self._events.append(event)
         self._exceeded_capacity = False
Beispiel #11
0
 def _post_to_inbox(self, message):
     try:
         self._inbox.put(message, block=False)
     except queue.Full:
         if not self._inbox_full:
             # possible race condition here, but it's of no real consequence - we'd just get an extra log line
             self._inbox_full = True
             log.warning("Events are being produced faster than they can be processed; some events will be dropped")
Beispiel #12
0
 def run(self):
     log.info("Starting event consumer")
     self._running = True
     while self._running:
         try:
             self.send()
         except Exception:
             log.warning('Unhandled exception in event consumer')
Beispiel #13
0
 def _send_event(self, event):
     if self._config.offline or not self._config.events_enabled:
         return
     event['creationDate'] = int(time.time() * 1000)
     if self._queue.full():
         log.warning("Event queue is full-- dropped an event")
     else:
         self._queue.put(event)
    def __init__(self, user, sdk_key=None, config=None, start_wait=5):
        """Constructs a new LDClient instance.
        :param string sdk_key: the SDK key for your LaunchDarkly environment
        :param ldclient.config.Config config: optional custom configuration
        :param float start_wait: the number of seconds to wait for a successful connection to LaunchDarkly
        """

        if user is None:
            raise Exception("user is required for mobile sdk")

        if config is not None and config.sdk_key is not None and sdk_key is not None:
            raise Exception(
                "LaunchDarkly client init received both sdk_key and config with sdk_key. "
                "Only one of either is expected")

        if sdk_key is not None:
            log.warning(
                "Deprecated sdk_key argument was passed to init. Use config object instead."
            )
            self._config = MobileConfig(sdk_key=sdk_key)
        else:
            self._config = config.copy_with_new_user(
                user) or MobileConfig.default().copy_with_new_user(user=user)

        self._config._validate()

        self._event_processor = None
        self._lock = Lock()
        self._event_factory_default = _EventFactory(False)
        self._event_factory_with_reasons = _EventFactory(True)

        self._store = _FeatureStoreClientWrapper(self._config.feature_store)
        """ :type: FeatureStore """

        if self._config.offline:
            log.info("Started LaunchDarkly Client in offline mode")

        if self._config.use_ldd:
            log.info("Started LaunchDarkly Client in LDD mode")

        self._event_processor = self._make_event_processor(self._config)
        update_processor_ready = threading.Event()
        self._update_processor = self._make_update_processor(
            self._config, self._store, update_processor_ready)
        self._update_processor.start()

        if start_wait > 0 and not self._config.offline and not self._config.use_ldd:
            log.info("Waiting up to " + str(start_wait) +
                     " seconds for LaunchDarkly client to initialize...")
            update_processor_ready.wait(start_wait)

        if self._update_processor.initialized() is True:
            log.info("Started LaunchDarkly Client: OK")

        else:
            log.warning(
                "Initialization timeout exceeded for LaunchDarkly Client or an error occurred. "
                "Feature Flags may not yet be available.")
Beispiel #15
0
 def _send(self, event):
     if self._offline or not self._config.events:
         return
     self._check_consumer()
     event["creationDate"] = int(time.time() * 1000)
     if self._queue.full():
         log.warning("Event queue is full-- dropped an event")
     else:
         self._queue.put(event)
Beispiel #16
0
 def _send(self, event):
     if self._offline or not self._config.events:
         return
     self._check_consumer()
     event['creationDate'] = int(time.time() * 1000)
     if self._queue.full():
         log.warning("Event queue is full-- dropped an event")
     else:
         self._queue.put(event)
Beispiel #17
0
 def add_event(self, event):
     if len(self._events) >= self._capacity:
         self._dropped_events += 1
         if not self._exceeded_capacity:
             log.warning("Exceeded event queue capacity. Increase capacity to avoid dropping events.")
             self._exceeded_capacity = True
     else:
         self._events.append(event)
         self._exceeded_capacity = False
 def _run_worker(self):
     while True:
         item = self._job_queue.get(block = True)
         if item is 'stop':
             return
         try:
             item()
         except Exception:
             log.warning('Unhandled exception in worker thread', exc_info=True)
         with self._lock:
             self._busy_count = self._busy_count - 1
             self._event.set()
Beispiel #19
0
 def run(self):
     # noinspection PyBroadException
     try:
         json_body = json.dumps(self._event_body)
         log.debug('Sending diagnostic event: ' + json_body)
         _post_events_with_retry(
             self._http, self._config,
             self._config.events_base_uri + '/diagnostic', None, json_body,
             "diagnostic event")
     except Exception as e:
         log.warning(
             'Unhandled exception in event processor. Diagnostic event was not sent. [%s]',
             e)
Beispiel #20
0
 def _run_worker(self):
     while True:
         item = self._job_queue.get(block=True)
         if item is 'stop':
             return
         try:
             item()
         except Exception:
             log.warning('Unhandled exception in worker thread',
                         exc_info=True)
         with self._lock:
             self._busy_count = self._busy_count - 1
             self._event.set()
Beispiel #21
0
    def identify(self, user):
        """Registers the user.

        This simply creates an analytics event that will transmit the given user properties to
        LaunchDarkly, so that the user will be visible on your dashboard even if you have not
        evaluated any flags for that user. It has no other effect.

        :param dict user: attributes of the user to register
        """
        if user is None or user.get('key') is None:
            log.warning("Missing user or user key when calling identify().")
        else:
            self._send_event(
                self._event_factory_default.new_identify_event(user))
 def run(self):
     # noinspection PyBroadException
     try:
         json_body = json.dumps(self._event_body)
         log.debug('Sending diagnostic event: ' + json_body)
         hdrs = _headers(self._config)
         uri = self._config.events_base_uri + '/diagnostic'
         r = self._http.request('POST', uri,
                                headers=hdrs,
                                timeout=urllib3.Timeout(connect=self._config.connect_timeout, read=self._config.read_timeout),
                                body=json_body,
                                retries=1)
     except Exception as e:
         log.warning(
             'Unhandled exception in event processor. Diagnostic event was not sent. [%s]', e)
Beispiel #23
0
 def process_message(store, msg):
     payload = json.loads(msg.data)
     if msg.event == 'put':
         store.init(payload)
     elif msg.event == 'patch':
         key = payload['path'][1:]
         feature = payload['data']
         store.upsert(key, feature)
     elif msg.event == 'delete':
         key = payload['path'][1:]
         # noinspection PyShadowingNames
         version = payload['version']
         store.delete(key, version)
     else:
         log.warning('Unhandled event in stream processor: ' + msg.event)
Beispiel #24
0
 def process_message(store, msg):
     payload = json.loads(msg.data)
     if msg.event == 'put':
         store.init(payload)
     elif msg.event == 'patch':
         key = payload['path'][1:]
         feature = payload['data']
         store.upsert(key, feature)
     elif msg.event == 'delete':
         key = payload['path'][1:]
         # noinspection PyShadowingNames
         version = payload['version']
         store.delete(key, version)
     else:
         log.warning('Unhandled event in stream processor: ' + msg.event)
Beispiel #25
0
 def _start_auto_updater(self):
     resolved_paths = []
     for path in self._paths:
         try:
             resolved_paths.append(os.path.realpath(path))
         except:
             log.warning(
                 'Cannot watch for changes to data file "%s" because it is an invalid path'
                 % path)
     if have_watchdog and not self._force_polling:
         return _FileDataSource.WatchdogAutoUpdater(resolved_paths,
                                                    self._load_all)
     else:
         return _FileDataSource.PollingAutoUpdater(resolved_paths,
                                                   self._load_all,
                                                   self._poll_interval)
Beispiel #26
0
 def do_toggle(should_retry):
     # noinspection PyBroadException,PyUnresolvedReferences
     try:
         val = self._toggle(key)
         return val
     except ProtocolError as e:
         inner = e.args[1]
         if inner.errno == errno.ECONNRESET and should_retry:
             log.warning('ProtocolError exception caught while getting flag. Retrying.')
             return do_toggle(False)
         else:
             log.exception('Unhandled exception. Returning default value for flag.')
             return None
     except Exception:
         log.exception('Unhandled exception. Returning default value for flag.')
         return None
Beispiel #27
0
 def _do_send(self, output_events):
     # noinspection PyBroadException
     try:
         json_body = json.dumps(output_events)
         log.debug('Sending events payload: ' + json_body)
         payload_id = str(uuid.uuid4())
         r = _post_events_with_retry(
             self._http, self._config, self._config.events_uri, payload_id,
             json_body, "%d events" % len(self._payload.events))
         if r:
             self._response_fn(r)
         return r
     except Exception as e:
         log.warning(
             'Unhandled exception in event processor. Analytics events were not processed. [%s]',
             e)
Beispiel #28
0
 def run(should_retry):
     # noinspection PyBroadException
     try:
         val = yield self._toggle(key)
         defer.returnValue(val)
     except ProtocolError as e:
         inner = e.args[1]
         if inner.errno == errno.ECONNRESET and should_retry:
             log.warning('ProtocolError exception caught while getting flag. Retrying.')
             d = yield run(False)
             defer.returnValue(d)
         else:
             log.exception('Unhandled exception. Returning default value for flag.')
             defer.returnValue(None)
     except Exception:
         log.exception('Unhandled exception. Returning default value for flag.')
         defer.returnValue(None)
Beispiel #29
0
 def _do_send(self, output_events):
     # noinspection PyBroadException
     try:
         json_body = json.dumps(output_events)
         log.debug('Sending events payload: ' + json_body)
         hdrs = _headers(self._config.sdk_key)
         hdrs['X-LaunchDarkly-Event-Schema'] = str(__CURRENT_EVENT_SCHEMA__)
         uri = self._config.events_uri
         r = self._http.request('POST', uri,
                                headers=hdrs,
                                timeout=urllib3.Timeout(connect=self._config.connect_timeout, read=self._config.read_timeout),
                                body=json_body,
                                retries=1)
         self._response_fn(r)
         return r
     except Exception as e:
         log.warning(
             'Unhandled exception in event processor. Analytics events were not processed. [%s]', e)
Beispiel #30
0
    def __init__(self, config: Config, start_wait: float = 5):
        """Constructs a new LDClient instance.

        :param config: optional custom configuration
        :param start_wait: the number of seconds to wait for a successful connection to LaunchDarkly
        """
        check_uwsgi()

        self._config = config
        self._config._validate()

        self._event_processor = None
        self._lock = Lock()
        self._event_factory_default = _EventFactory(False)
        self._event_factory_with_reasons = _EventFactory(True)

        self._store = _FeatureStoreClientWrapper(self._config.feature_store)
        """ :type: FeatureStore """

        if self._config.offline:
            log.info("Started LaunchDarkly Client in offline mode")

        if self._config.use_ldd:
            log.info("Started LaunchDarkly Client in LDD mode")

        diagnostic_accumulator = self._set_event_processor(self._config)

        update_processor_ready = threading.Event()
        self._update_processor = self._make_update_processor(
            self._config, self._store, update_processor_ready,
            diagnostic_accumulator)
        self._update_processor.start()

        if start_wait > 0 and not self._config.offline and not self._config.use_ldd:
            log.info("Waiting up to " + str(start_wait) +
                     " seconds for LaunchDarkly client to initialize...")
            update_processor_ready.wait(start_wait)

        if self._update_processor.initialized() is True:
            log.info("Started LaunchDarkly Client: OK")
        else:
            log.warning(
                "Initialization timeout exceeded for LaunchDarkly Client or an error occurred. "
                "Feature Flags may not yet be available.")
 def track(self, event_name, data=None, metric_value=None):
     """Tracks that a user performed an event.
     LaunchDarkly automatically tracks pageviews and clicks that are specified in the Goals
     section of the dashboard. This can be used to track custom goals or other events that do
     not currently have goals.
     :param string event_name: the name of the event, which may correspond to a goal in A/B tests
     :param dict user: the attributes of the user
     :param data: optional additional data associated with the event
     :param metric_value: a numeric value used by the LaunchDarkly experimentation feature in
       numeric custom metrics. Can be omitted if this event is used by only non-numeric metrics.
       This field will also be returned as part of the custom event for Data Export.
     """
     user = self._config.user
     if user is None or user.get('key') is None:
         log.warning("Missing user or user key when calling track().")
     else:
         self._send_event(
             self._event_factory_default.new_custom_event(
                 event_name, user, data, metric_value))
Beispiel #32
0
    def parse(cls, raw, instance=None):
        """
        Given a possibly-multiline string representing an SSE message, parse it
        and return a Event object.
        """
        msg = cls()
        for line in raw.split('\n'):
            m = cls.sse_line_pattern.match(line)
            if m is None:
                # Malformed line.  Discard but warn.
                log.warning('Invalid SSE line: "%s"' % line)
                continue

            name = m.groupdict()['name']
            value = m.groupdict()['value']
            if name == '':
                if instance is not None:
                    now = time.time()
                    last_heartbeat = instance._last_heartbeat or instance._connect_start
                    duration = int((now - last_heartbeat) * 1000)
                    request_success.fire(request_type='sse:heartbeat',
                                         name=clean_name('GET', instance.url),
                                         response_time=duration,
                                         response_length=0)
                    instance._last_heartbeat = now
                # line began with a ":", so is a comment.  Ignore
                continue

            if name == 'data':
                # If we already have some data, then join to it with a newline.
                # Else this is it.
                if msg.data:
                    msg.data = '%s\n%s' % (msg.data, value)
                else:
                    msg.data = value
            elif name == 'event':
                msg.event = value
            elif name == 'id':
                msg.id = value
            elif name == 'retry':
                msg.retry = int(value)

        return msg
Beispiel #33
0
    def process_message(store, requester, msg):
        if msg.event == 'put':
            all_data = json.loads(msg.data)

            for k,v in all_data.items():
                v['key'] = k

            init_data = {
                FEATURES: all_data
            }
            log.debug("Received put event with %d flags",
                len(init_data[FEATURES]))
            store.init(init_data)
            return True
        elif msg.event == 'patch':
            recv_time = time.time() * 1000
            payload = json.loads(msg.data)
            if payload.get('key') == 'locust-heartbeat':
                value = int(payload.get('value') or 0)
                duration = int((recv_time - value))
                request_success.fire(request_type='sse:flag-update', name='/meval', response_time=duration, response_length=0)
                
            log.debug("Received patch event for %s, New version: [%d]", payload.get('key'), payload.get("version"))
            store.upsert(FEATURES, payload)
        elif msg.event == 'ping':
            log.debug('Received ping event')
            all_data = requester.get_all_data()
            store.init(all_data)
            log.debug("Received flags after ping event with %d flags", len(all_data[FEATURES]))
            return True
            
        elif msg.event == 'delete':
            payload = json.loads(msg.data)
            key = payload.get('key')
            # noinspection PyShadowingNames
            version = payload['version']
            log.debug("Received delete event for %s, New version: [%d]", key, version)
            target = ParsedPath(kind = FEATURES, key = key)
            store.delete(target.kind, target.key, version)
        else:
            log.warning('Unhandled event in stream processor: ' + msg.event)
        return False
Beispiel #34
0
 def do_send(should_retry):
     # noinspection PyBroadException
     try:
         if isinstance(events, dict):
             body = [events]
         else:
             body = events
         hdrs = _headers(self._api_key)
         uri = self._config.base_uri + '/api/events/bulk'
         r = self._session.post(uri, headers=hdrs, timeout=(self._config.connect, self._config.read),
                                data=json.dumps(body))
         r.raise_for_status()
     except ProtocolError as e:
         inner = e.args[1]
         if inner.errno == errno.ECONNRESET and should_retry:
             log.warning('ProtocolError exception caught while sending events. Retrying.')
             do_send(False)
         else:
             log.exception('Unhandled exception in event consumer. Analytics events were not processed.')
     except:
         log.exception('Unhandled exception in event consumer. Analytics events were not processed.')
Beispiel #35
0
 def process_message(store, requester, msg):
     if msg.event == 'put':
         all_data = json.loads(msg.data)
         init_data = {
             FEATURES: all_data['data']['flags'],
             SEGMENTS: all_data['data']['segments']
         }
         log.debug("Received put event with %d flags and %d segments",
                   len(init_data[FEATURES]), len(init_data[SEGMENTS]))
         store.init(init_data)
         return True
     elif msg.event == 'patch':
         payload = json.loads(msg.data)
         path = payload['path']
         obj = payload['data']
         log.debug("Received patch event for %s, New version: [%d]", path,
                   obj.get("version"))
         target = StreamingUpdateProcessor._parse_path(path)
         if target is not None:
             store.upsert(target.kind, obj)
         else:
             log.warning("Patch for unknown path: %s", path)
     elif msg.event == "indirect/patch":
         path = msg.data
         log.debug("Received indirect/patch event for %s", path)
         target = StreamingUpdateProcessor._parse_path(path)
         if target is not None:
             store.upsert(target.kind,
                          requester.get_one(target.kind, target.key))
         else:
             log.warning("Indirect patch for unknown path: %s", path)
     elif msg.event == "indirect/put":
         log.debug("Received indirect/put event")
         store.init(requester.get_all_data())
         return True
     elif msg.event == 'delete':
         payload = json.loads(msg.data)
         path = payload['path']
         # noinspection PyShadowingNames
         version = payload['version']
         log.debug("Received delete event for %s, New version: [%d]", path,
                   version)
         target = StreamingUpdateProcessor._parse_path(path)
         if target is not None:
             store.delete(target.kind, target.key, version)
         else:
             log.warning("Delete for unknown path: %s", path)
     else:
         log.warning('Unhandled event in stream processor: ' + msg.event)
     return False
Beispiel #36
0
 def run(self):
     log.info("Starting StreamingUpdateProcessor connecting to uri: " + self._uri)
     self._running = True
     while self._running:
         try:
             messages = self._connect()
             for msg in messages:
                 if not self._running:
                     break
                 message_ok = self.process_message(self._store, self._requester, msg)
                 if message_ok is True and self._ready.is_set() is False:
                     log.info("StreamingUpdateProcessor initialized ok.")
                     self._ready.set()
         except UnsuccessfulResponseException as e:
             log.error(http_error_message(e.status, "stream connection"))
             if not is_http_error_recoverable(e.status):
                 self._ready.set()  # if client is initializing, make it stop waiting; has no effect if already inited
                 self.stop()
                 break
         except Exception as e:
             log.warning("Caught exception. Restarting stream connection after one second. %s" % e)
             # no stacktrace here because, for a typical connection error, it'll just be a lengthy tour of urllib3 internals
         time.sleep(1)
Beispiel #37
0
 def process_message(store, requester, msg):
     if msg.event == 'put':
         all_data = json.loads(msg.data)
         init_data = {
             FEATURES: all_data['data']['flags'],
             SEGMENTS: all_data['data']['segments']
         }
         log.debug("Received put event with %d flags and %d segments",
             len(init_data[FEATURES]), len(init_data[SEGMENTS]))
         store.init(init_data)
         return True
     elif msg.event == 'patch':
         payload = json.loads(msg.data)
         path = payload['path']
         obj = payload['data']
         log.debug("Received patch event for %s, New version: [%d]", path, obj.get("version"))
         target = StreamingUpdateProcessor._parse_path(path)
         if target is not None:
             store.upsert(target.kind, obj)
         else:
             log.warning("Patch for unknown path: %s", path)
     elif msg.event == "indirect/patch":
         path = msg.data
         log.debug("Received indirect/patch event for %s", path)
         target = StreamingUpdateProcessor._parse_path(path)
         if target is not None:
             store.upsert(target.kind, requester.get_one(target.kind, target.key))
         else:
             log.warning("Indirect patch for unknown path: %s", path)
     elif msg.event == "indirect/put":
         log.debug("Received indirect/put event")
         store.init(requester.get_all_data())
         return True
     elif msg.event == 'delete':
         payload = json.loads(msg.data)
         path = payload['path']
         # noinspection PyShadowingNames
         version = payload['version']
         log.debug("Received delete event for %s, New version: [%d]", path, version)
         target = StreamingUpdateProcessor._parse_path(path)
         if target is not None:
             store.delete(target.kind, target.key, version)
         else:
             log.warning("Delete for unknown path: %s", path)
     else:
         log.warning('Unhandled event in stream processor: ' + msg.event)
     return False