Example #1
0
async def _queue_pattern_ltd_product_ingest(
    *,
    session: ClientSession,
    logger: BoundLoggerLazyProxy,
    config: Configuration,
    producer: AIOKafkaProducer,
    schema_manager: RecordNameSchemaManager,
    product_pattern: str,
    edition_slug: str,
) -> None:
    """Queue a LTD-based documents with product slugs matching a regular
    expression pattern for ingest in the ook.ingest Kafka topic.
    """
    product_data = await _get_json(session=session,
                                   url="https://keeper.lsst.codes/products/")
    url_prefix = "https://keeper.lsst.codes/products/"
    all_products = [p[len(url_prefix):] for p in product_data["products"]]
    pattern = re.compile(product_pattern)
    matching_products = [
        p for p in all_products if pattern.match(p) is not None
    ]
    logger.info("Matched products", product_slugs=matching_products)
    await _queue_list_ltd_product_ingest(
        session=session,
        logger=logger,
        config=config,
        producer=producer,
        schema_manager=schema_manager,
        product_slugs=matching_products,
        edition_slug=edition_slug,
    )
Example #2
0
async def _queue_list_ltd_product_ingest(
    *,
    session: ClientSession,
    logger: BoundLoggerLazyProxy,
    config: Configuration,
    producer: AIOKafkaProducer,
    schema_manager: RecordNameSchemaManager,
    product_slugs: List[str],
    edition_slug: str,
) -> None:
    """Queue a list of LTD-based documents (with known product slugs) for
    ingest in the ook.ingest Kafka topic.
    """
    for product_slug in product_slugs:
        try:
            await _queue_single_ltd_product_ingest(
                session=session,
                logger=logger,
                config=config,
                producer=producer,
                schema_manager=schema_manager,
                product_slug=product_slug,
                edition_slug=edition_slug,
            )
        except Exception:
            logger.exception(
                "Failed to queue LTD product ingest",
                product_slug=product_slug,
                edition_slug=edition_slug,
            )
Example #3
0
    def test_honors_wrapper_class(self):
        """
        Passed wrapper_class is used.
        """
        p = BoundLoggerLazyProxy(None, wrapper_class=Wrapper)
        b = p.bind()

        assert isinstance(b, Wrapper)
Example #4
0
    def test_unbind_unbinds_from_initial_values(self):
        """
        It's possible to unbind a value that came from initial_values.
        """
        p = BoundLoggerLazyProxy(None, initial_values={"a": 1, "b": 2})
        b = p.unbind("a")

        assert {"b": 2} == b._context
Example #5
0
    def test_new_binds_only_initial_values_implicit_ctx_class(self, proxy):
        """
        new() doesn't clear initial_values if context_class comes from config.
        """
        proxy = BoundLoggerLazyProxy(None, initial_values={"a": 1, "b": 2})

        b = proxy.new(foo=42)

        assert {"a": 1, "b": 2, "foo": 42} == b._context
Example #6
0
 def test_new_binds_only_initial_values_explicit_ctx_class(self, proxy):
     proxy = BoundLoggerLazyProxy(None,
                                  initial_values={
                                      "a": 1,
                                      "b": 2
                                  },
                                  context_class=dict)
     b = proxy.new(foo=42)
     assert {"a": 1, "b": 2, "foo": 42} == b._context
Example #7
0
    def test_bind_honors_initial_values(self):
        """
        Passed initia_values are merged on binds.
        """
        p = BoundLoggerLazyProxy(None, initial_values={"a": 1, "b": 2})
        b = p.bind()

        assert {"a": 1, "b": 2} == b._context

        b = p.bind(c=3)

        assert {"a": 1, "b": 2, "c": 3} == b._context
Example #8
0
    def test_emphemeral(self):
        """
        Calling an unknown method proxy creates a new wrapped bound logger
        first.
        """

        class Foo(BoundLoggerBase):
            def foo(self):
                return 42

        proxy = BoundLoggerLazyProxy(None, wrapper_class=Foo, cache_logger_on_first_use=False)
        assert 42 == proxy.foo()
Example #9
0
    def test_emphemeral(self):
        """
        Calling an unknown method proxy creates a new wrapped bound logger
        first.
        """
        class Foo(BoundLoggerBase):
            def foo(self):
                return 42

        proxy = BoundLoggerLazyProxy(None,
                                     wrapper_class=Foo,
                                     cache_logger_on_first_use=False)
        assert 42 == proxy.foo()
Example #10
0
 def test_new_binds_only_initial_values_explicit_ctx_class(self, proxy):
     """
     new() doesn't clear initial_values if context_class is passed
     explicitly..
     """
     proxy = BoundLoggerLazyProxy(None,
                                  initial_values={
                                      "a": 1,
                                      "b": 2
                                  },
                                  context_class=dict)
     b = proxy.new(foo=42)
     assert {"a": 1, "b": 2, "foo": 42} == b._context
Example #11
0
    def test_argument_takes_precedence_over_configuration(self, cache):
        """
        Passing cache_logger_on_first_use as an argument overrides config.
        """
        configure(cache_logger_on_first_use=cache)

        proxy = BoundLoggerLazyProxy(None, cache_logger_on_first_use=not cache)
        bind = proxy.bind
        proxy.bind()

        if cache:
            assert bind == proxy.bind
        else:
            assert bind != proxy.bind
Example #12
0
    def test_bind_doesnt_cache_logger(self):
        """
        Calling configure() changes BoundLoggerLazyProxys immediately.
        Previous uses of the BoundLoggerLazyProxy don't interfere.
        """
        class F(object):
            "New logger factory with a new attribute"
            def a(self, *args):
                return 5

        proxy = BoundLoggerLazyProxy(None)
        proxy.bind()
        configure(logger_factory=F)
        new_b = proxy.bind()
        assert new_b.a() == 5
Example #13
0
    def test_pickle(self, proto):
        """
        Can be pickled and unpickled.
        """
        bllp = BoundLoggerLazyProxy(None)

        assert repr(bllp) == repr(pickle.loads(pickle.dumps(bllp, proto)))
Example #14
0
    def test_prefers_args_over_config(self):
        p = BoundLoggerLazyProxy(None, processors=[1, 2, 3],
                                 context_class=dict)
        b = p.bind()
        assert isinstance(b._context, dict)
        assert [1, 2, 3] == b._processors

        class Class(object):
            def __init__(self, *args, **kw):
                pass

            def update(self, *args, **kw):
                pass
        configure(processors=[4, 5, 6], context_class=Class)
        b = p.bind()
        assert not isinstance(b._context, Class)
        assert [1, 2, 3] == b._processors
Example #15
0
    def test_prefers_args_over_config(self):
        p = BoundLoggerLazyProxy(None, processors=[1, 2, 3], context_class=dict)
        b = p.bind()
        assert isinstance(b._context, dict)
        assert [1, 2, 3] == b._processors

        class Class(object):
            def __init__(self, *args, **kw):
                pass

            def update(self, *args, **kw):
                pass

        configure(processors=[4, 5, 6], context_class=Class)
        b = p.bind()
        assert not isinstance(b._context, Class)
        assert [1, 2, 3] == b._processors
Example #16
0
async def send_tasks_to_worker(log: BoundLoggerLazyProxy, redis_app: Redis,
                               arg: Optional[str], now: datetime,
                               timestamp: Optional[datetime], timeout: int,
                               handler_type: str, lang: str) -> None:
    # updates conditions
    condition = timestamp is None or (now -
                                      timestamp).total_seconds() > timeout

    # make updates (send tasks to worker)
    if handler_type == 'static_category' and condition:
        log.info('Send task to worker',
                 stream=REDIS_STREAM_CHALLENGES,
                 lang=lang)
        await redis_app.xadd(REDIS_STREAM_CHALLENGES, {
            b'lang': lang.encode(),
            b'update': b"ok"
        })
    elif handler_type == 'dynamic_user' and arg is not None and condition:
        await redis_app.xadd(REDIS_STREAM_USERS, {
            b'lang': lang.encode(),
            b'username': arg.encode()
        })
        log.info('Send task to worker',
                 stream=REDIS_STREAM_USERS,
                 username=arg,
                 lang=lang)
    elif handler_type == 'dynamic_categories' and arg is not None and condition:
        await redis_app.xadd(REDIS_STREAM_CHALLENGES, {
            b'lang': lang.encode(),
            b'update': b"ok"
        })
        log.info('Send task to worker',
                 stream=REDIS_STREAM_CHALLENGES,
                 lang=lang)
Example #17
0
 def test_repr(self):
     p = BoundLoggerLazyProxy(
         None, processors=[1, 2, 3], context_class=dict,
         initial_values={'foo': 42}, logger_factory_args=(4, 5),
     )
     assert (
         "<BoundLoggerLazyProxy(logger=None, wrapper_class=None, "
         "processors=[1, 2, 3], "
         "context_class=<%s 'dict'>, "
         "initial_values={'foo': 42}, "
         "logger_factory_args=(4, 5))>"
         % ('class' if PY3 else 'type',)
     ) == repr(p)
Example #18
0
    def test_prefers_args_over_config(self):
        """
        Configuration can be overridden by passing arguments.
        """
        p = BoundLoggerLazyProxy(None,
                                 processors=[1, 2, 3],
                                 context_class=dict)
        b = p.bind()
        assert isinstance(b._context, dict)
        assert [1, 2, 3] == b._processors

        class Class:
            def __init__(self, *args, **kw):
                pass

            def update(self, *args, **kw):
                pass

        configure(processors=[4, 5, 6], context_class=Class)
        b = p.bind()

        assert not isinstance(b._context, Class)
        assert [1, 2, 3] == b._processors
Example #19
0
 def test_repr(self):
     """
     repr reflects all attributes.
     """
     p = BoundLoggerLazyProxy(
         None,
         processors=[1, 2, 3],
         context_class=dict,
         initial_values={"foo": 42},
         logger_factory_args=(4, 5),
     )
     assert ("<BoundLoggerLazyProxy(logger=None, wrapper_class=None, "
             "processors=[1, 2, 3], "
             "context_class=<class 'dict'>, "
             "initial_values={'foo': 42}, "
             "logger_factory_args=(4, 5))>") == repr(p)
Example #20
0
 def test_unbind_unbinds_from_initial_values(self):
     p = BoundLoggerLazyProxy(None, initial_values={"a": 1, "b": 2})
     b = p.unbind("a")
     assert {"b": 2} == b._context
Example #21
0
 def test_bind_honors_initial_values(self):
     p = BoundLoggerLazyProxy(None, initial_values={"a": 1, "b": 2})
     b = p.bind()
     assert {"a": 1, "b": 2} == b._context
     b = p.bind(c=3)
     assert {"a": 1, "b": 2, "c": 3} == b._context
Example #22
0
async def _queue_single_ltd_product_ingest(
    *,
    session: ClientSession,
    logger: BoundLoggerLazyProxy,
    config: Configuration,
    producer: AIOKafkaProducer,
    schema_manager: RecordNameSchemaManager,
    product_slug: str,
    edition_slug: str,
) -> None:
    """Queue an LTD-based document for ingest in the ook.ingest Kafka topic."""
    product_data = await _get_json(
        session=session,
        url=f"https://keeper.lsst.codes/products/{product_slug}",
    )
    edition_urls = await _get_json(
        session=session,
        url=f"https://keeper.lsst.codes/products/{product_slug}/editions/",
    )
    for edition_url in edition_urls["editions"]:
        edition_data = await _get_json(session=session, url=edition_url)
        if edition_data["slug"] == edition_slug:
            break
    if edition_data["slug"] != edition_slug:
        raise RuntimeError(
            "Could not find slug {edition_slug} for product {product_slug}")

    content_type = await classify_ltd_site(
        http_session=session,
        product_slug=product_slug,
        published_url=edition_data["published_url"],
    )
    ltd_document_types = {
        ContentType.LTD_LANDER_JSONLD,
        ContentType.LTD_SPHINX_TECHNOTE,
    }
    if content_type not in ltd_document_types:
        logger.warning(
            "Cannot do triggered ingest of a non-document "
            "LTD product.",
            content_type=content_type.name,
        )
        return

    key = {"url": edition_data["published_url"]}
    value = {
        "content_type": content_type.name,
        "request_timestamp": datetime.datetime.utcnow(),
        "update_timestamp": datetime.datetime.utcnow(),
        "url": edition_data["published_url"],
        "edition": {
            "url": edition_data["self_url"],
            "published_url": edition_data["published_url"],
            "slug": edition_slug,
            "build_url": edition_data["build_url"],
        },
        "product": {
            "url": product_data["self_url"],
            "published_url": edition_data["published_url"],
            "slug": product_slug,
        },
    }
    key_data = await schema_manager.serialize(data=key, name="ook.url_key_v1")
    value_data = await schema_manager.serialize(data=value,
                                                name="ook.ltd_url_ingest_v1")
    # Produce message
    topic_name = config.ingest_kafka_topic
    await producer.send_and_wait(topic_name, key=key_data, value=value_data)
    logger.info(
        "Produced an LTD document URL ingest request",
        topic=topic_name,
        url=value["url"],
    )
Example #23
0
 def test_argument_takes_precedence_over_configuration2(self):
     configure(cache_logger_on_first_use=False)
     proxy = BoundLoggerLazyProxy(None, cache_logger_on_first_use=True)
     bind = proxy.bind
     proxy.bind()
     assert bind != proxy.bind
Example #24
0
 def test_new_binds_only_initial_values_explicit_ctx_class(self, proxy):
     proxy = BoundLoggerLazyProxy(None, initial_values={"a": 1, "b": 2}, context_class=dict)
     b = proxy.new(foo=42)
     assert {"a": 1, "b": 2, "foo": 42} == b._context
Example #25
0
 def test_honors_wrapper_class(self):
     p = BoundLoggerLazyProxy(None, wrapper_class=Wrapper)
     b = p.bind()
     assert isinstance(b, Wrapper)
Example #26
0
 def test_unbind_unbinds_from_initial_values(self):
     p = BoundLoggerLazyProxy(None, initial_values={"a": 1, "b": 2})
     b = p.unbind("a")
     assert {"b": 2} == b._context
Example #27
0
 def test_bind_honors_initial_values(self):
     p = BoundLoggerLazyProxy(None, initial_values={"a": 1, "b": 2})
     b = p.bind()
     assert {"a": 1, "b": 2} == b._context
     b = p.bind(c=3)
     assert {"a": 1, "b": 2, "c": 3} == b._context
Example #28
0
 def test_new_binds_only_initial_values_impolicit_ctx_class(self, proxy):
     proxy = BoundLoggerLazyProxy(None, initial_values={'a': 1, 'b': 2})
     b = proxy.new(foo=42)
     assert {'a': 1, 'b': 2, 'foo': 42} == b._context
Example #29
0
 def test_unbind_unbinds_from_initial_values(self):
     p = BoundLoggerLazyProxy(None, initial_values={'a': 1, 'b': 2})
     b = p.unbind('a')
     assert {'b': 2} == b._context
Example #30
0
 def test_bind_honors_initial_values(self):
     p = BoundLoggerLazyProxy(None, initial_values={'a': 1, 'b': 2})
     b = p.bind()
     assert {'a': 1, 'b': 2} == b._context
     b = p.bind(c=3)
     assert {'a': 1, 'b': 2, 'c': 3} == b._context
Example #31
0
 def test_new_binds_only_initial_values_impolicit_ctx_class(self, proxy):
     proxy = BoundLoggerLazyProxy(None, initial_values={'a': 1, 'b': 2})
     b = proxy.new(foo=42)
     assert {'a': 1, 'b': 2, 'foo': 42} == b._context
Example #32
0
 def test_argument_takes_precedence_over_configuration2(self):
     configure(cache_logger_on_first_use=False)
     proxy = BoundLoggerLazyProxy(None, cache_logger_on_first_use=True)
     bind = proxy.bind
     proxy.bind()
     assert bind != proxy.bind
Example #33
0
 def test_bind_honors_initial_values(self):
     p = BoundLoggerLazyProxy(None, initial_values={'a': 1, 'b': 2})
     b = p.bind()
     assert {'a': 1, 'b': 2} == b._context
     b = p.bind(c=3)
     assert {'a': 1, 'b': 2, 'c': 3} == b._context
Example #34
0
def proxy():
    """
    Returns a BoundLoggerLazyProxy constructed w/o paramaters & None as logger.
    """
    return BoundLoggerLazyProxy(None)
Example #35
0
 def test_unbind_unbinds_from_initial_values(self):
     p = BoundLoggerLazyProxy(None, initial_values={'a': 1, 'b': 2})
     b = p.unbind('a')
     assert {'b': 2} == b._context