async def _queue_pattern_ltd_product_ingest( *, session: ClientSession, logger: BoundLoggerLazyProxy, config: Configuration, producer: AIOKafkaProducer, schema_manager: RecordNameSchemaManager, product_pattern: str, edition_slug: str, ) -> None: """Queue a LTD-based documents with product slugs matching a regular expression pattern for ingest in the ook.ingest Kafka topic. """ product_data = await _get_json(session=session, url="https://keeper.lsst.codes/products/") url_prefix = "https://keeper.lsst.codes/products/" all_products = [p[len(url_prefix):] for p in product_data["products"]] pattern = re.compile(product_pattern) matching_products = [ p for p in all_products if pattern.match(p) is not None ] logger.info("Matched products", product_slugs=matching_products) await _queue_list_ltd_product_ingest( session=session, logger=logger, config=config, producer=producer, schema_manager=schema_manager, product_slugs=matching_products, edition_slug=edition_slug, )
async def _queue_list_ltd_product_ingest( *, session: ClientSession, logger: BoundLoggerLazyProxy, config: Configuration, producer: AIOKafkaProducer, schema_manager: RecordNameSchemaManager, product_slugs: List[str], edition_slug: str, ) -> None: """Queue a list of LTD-based documents (with known product slugs) for ingest in the ook.ingest Kafka topic. """ for product_slug in product_slugs: try: await _queue_single_ltd_product_ingest( session=session, logger=logger, config=config, producer=producer, schema_manager=schema_manager, product_slug=product_slug, edition_slug=edition_slug, ) except Exception: logger.exception( "Failed to queue LTD product ingest", product_slug=product_slug, edition_slug=edition_slug, )
def test_honors_wrapper_class(self): """ Passed wrapper_class is used. """ p = BoundLoggerLazyProxy(None, wrapper_class=Wrapper) b = p.bind() assert isinstance(b, Wrapper)
def test_unbind_unbinds_from_initial_values(self): """ It's possible to unbind a value that came from initial_values. """ p = BoundLoggerLazyProxy(None, initial_values={"a": 1, "b": 2}) b = p.unbind("a") assert {"b": 2} == b._context
def test_new_binds_only_initial_values_implicit_ctx_class(self, proxy): """ new() doesn't clear initial_values if context_class comes from config. """ proxy = BoundLoggerLazyProxy(None, initial_values={"a": 1, "b": 2}) b = proxy.new(foo=42) assert {"a": 1, "b": 2, "foo": 42} == b._context
def test_new_binds_only_initial_values_explicit_ctx_class(self, proxy): proxy = BoundLoggerLazyProxy(None, initial_values={ "a": 1, "b": 2 }, context_class=dict) b = proxy.new(foo=42) assert {"a": 1, "b": 2, "foo": 42} == b._context
def test_bind_honors_initial_values(self): """ Passed initia_values are merged on binds. """ p = BoundLoggerLazyProxy(None, initial_values={"a": 1, "b": 2}) b = p.bind() assert {"a": 1, "b": 2} == b._context b = p.bind(c=3) assert {"a": 1, "b": 2, "c": 3} == b._context
def test_emphemeral(self): """ Calling an unknown method proxy creates a new wrapped bound logger first. """ class Foo(BoundLoggerBase): def foo(self): return 42 proxy = BoundLoggerLazyProxy(None, wrapper_class=Foo, cache_logger_on_first_use=False) assert 42 == proxy.foo()
def test_new_binds_only_initial_values_explicit_ctx_class(self, proxy): """ new() doesn't clear initial_values if context_class is passed explicitly.. """ proxy = BoundLoggerLazyProxy(None, initial_values={ "a": 1, "b": 2 }, context_class=dict) b = proxy.new(foo=42) assert {"a": 1, "b": 2, "foo": 42} == b._context
def test_argument_takes_precedence_over_configuration(self, cache): """ Passing cache_logger_on_first_use as an argument overrides config. """ configure(cache_logger_on_first_use=cache) proxy = BoundLoggerLazyProxy(None, cache_logger_on_first_use=not cache) bind = proxy.bind proxy.bind() if cache: assert bind == proxy.bind else: assert bind != proxy.bind
def test_bind_doesnt_cache_logger(self): """ Calling configure() changes BoundLoggerLazyProxys immediately. Previous uses of the BoundLoggerLazyProxy don't interfere. """ class F(object): "New logger factory with a new attribute" def a(self, *args): return 5 proxy = BoundLoggerLazyProxy(None) proxy.bind() configure(logger_factory=F) new_b = proxy.bind() assert new_b.a() == 5
def test_pickle(self, proto): """ Can be pickled and unpickled. """ bllp = BoundLoggerLazyProxy(None) assert repr(bllp) == repr(pickle.loads(pickle.dumps(bllp, proto)))
def test_prefers_args_over_config(self): p = BoundLoggerLazyProxy(None, processors=[1, 2, 3], context_class=dict) b = p.bind() assert isinstance(b._context, dict) assert [1, 2, 3] == b._processors class Class(object): def __init__(self, *args, **kw): pass def update(self, *args, **kw): pass configure(processors=[4, 5, 6], context_class=Class) b = p.bind() assert not isinstance(b._context, Class) assert [1, 2, 3] == b._processors
async def send_tasks_to_worker(log: BoundLoggerLazyProxy, redis_app: Redis, arg: Optional[str], now: datetime, timestamp: Optional[datetime], timeout: int, handler_type: str, lang: str) -> None: # updates conditions condition = timestamp is None or (now - timestamp).total_seconds() > timeout # make updates (send tasks to worker) if handler_type == 'static_category' and condition: log.info('Send task to worker', stream=REDIS_STREAM_CHALLENGES, lang=lang) await redis_app.xadd(REDIS_STREAM_CHALLENGES, { b'lang': lang.encode(), b'update': b"ok" }) elif handler_type == 'dynamic_user' and arg is not None and condition: await redis_app.xadd(REDIS_STREAM_USERS, { b'lang': lang.encode(), b'username': arg.encode() }) log.info('Send task to worker', stream=REDIS_STREAM_USERS, username=arg, lang=lang) elif handler_type == 'dynamic_categories' and arg is not None and condition: await redis_app.xadd(REDIS_STREAM_CHALLENGES, { b'lang': lang.encode(), b'update': b"ok" }) log.info('Send task to worker', stream=REDIS_STREAM_CHALLENGES, lang=lang)
def test_repr(self): p = BoundLoggerLazyProxy( None, processors=[1, 2, 3], context_class=dict, initial_values={'foo': 42}, logger_factory_args=(4, 5), ) assert ( "<BoundLoggerLazyProxy(logger=None, wrapper_class=None, " "processors=[1, 2, 3], " "context_class=<%s 'dict'>, " "initial_values={'foo': 42}, " "logger_factory_args=(4, 5))>" % ('class' if PY3 else 'type',) ) == repr(p)
def test_prefers_args_over_config(self): """ Configuration can be overridden by passing arguments. """ p = BoundLoggerLazyProxy(None, processors=[1, 2, 3], context_class=dict) b = p.bind() assert isinstance(b._context, dict) assert [1, 2, 3] == b._processors class Class: def __init__(self, *args, **kw): pass def update(self, *args, **kw): pass configure(processors=[4, 5, 6], context_class=Class) b = p.bind() assert not isinstance(b._context, Class) assert [1, 2, 3] == b._processors
def test_repr(self): """ repr reflects all attributes. """ p = BoundLoggerLazyProxy( None, processors=[1, 2, 3], context_class=dict, initial_values={"foo": 42}, logger_factory_args=(4, 5), ) assert ("<BoundLoggerLazyProxy(logger=None, wrapper_class=None, " "processors=[1, 2, 3], " "context_class=<class 'dict'>, " "initial_values={'foo': 42}, " "logger_factory_args=(4, 5))>") == repr(p)
def test_unbind_unbinds_from_initial_values(self): p = BoundLoggerLazyProxy(None, initial_values={"a": 1, "b": 2}) b = p.unbind("a") assert {"b": 2} == b._context
def test_bind_honors_initial_values(self): p = BoundLoggerLazyProxy(None, initial_values={"a": 1, "b": 2}) b = p.bind() assert {"a": 1, "b": 2} == b._context b = p.bind(c=3) assert {"a": 1, "b": 2, "c": 3} == b._context
async def _queue_single_ltd_product_ingest( *, session: ClientSession, logger: BoundLoggerLazyProxy, config: Configuration, producer: AIOKafkaProducer, schema_manager: RecordNameSchemaManager, product_slug: str, edition_slug: str, ) -> None: """Queue an LTD-based document for ingest in the ook.ingest Kafka topic.""" product_data = await _get_json( session=session, url=f"https://keeper.lsst.codes/products/{product_slug}", ) edition_urls = await _get_json( session=session, url=f"https://keeper.lsst.codes/products/{product_slug}/editions/", ) for edition_url in edition_urls["editions"]: edition_data = await _get_json(session=session, url=edition_url) if edition_data["slug"] == edition_slug: break if edition_data["slug"] != edition_slug: raise RuntimeError( "Could not find slug {edition_slug} for product {product_slug}") content_type = await classify_ltd_site( http_session=session, product_slug=product_slug, published_url=edition_data["published_url"], ) ltd_document_types = { ContentType.LTD_LANDER_JSONLD, ContentType.LTD_SPHINX_TECHNOTE, } if content_type not in ltd_document_types: logger.warning( "Cannot do triggered ingest of a non-document " "LTD product.", content_type=content_type.name, ) return key = {"url": edition_data["published_url"]} value = { "content_type": content_type.name, "request_timestamp": datetime.datetime.utcnow(), "update_timestamp": datetime.datetime.utcnow(), "url": edition_data["published_url"], "edition": { "url": edition_data["self_url"], "published_url": edition_data["published_url"], "slug": edition_slug, "build_url": edition_data["build_url"], }, "product": { "url": product_data["self_url"], "published_url": edition_data["published_url"], "slug": product_slug, }, } key_data = await schema_manager.serialize(data=key, name="ook.url_key_v1") value_data = await schema_manager.serialize(data=value, name="ook.ltd_url_ingest_v1") # Produce message topic_name = config.ingest_kafka_topic await producer.send_and_wait(topic_name, key=key_data, value=value_data) logger.info( "Produced an LTD document URL ingest request", topic=topic_name, url=value["url"], )
def test_argument_takes_precedence_over_configuration2(self): configure(cache_logger_on_first_use=False) proxy = BoundLoggerLazyProxy(None, cache_logger_on_first_use=True) bind = proxy.bind proxy.bind() assert bind != proxy.bind
def test_new_binds_only_initial_values_explicit_ctx_class(self, proxy): proxy = BoundLoggerLazyProxy(None, initial_values={"a": 1, "b": 2}, context_class=dict) b = proxy.new(foo=42) assert {"a": 1, "b": 2, "foo": 42} == b._context
def test_honors_wrapper_class(self): p = BoundLoggerLazyProxy(None, wrapper_class=Wrapper) b = p.bind() assert isinstance(b, Wrapper)
def test_new_binds_only_initial_values_impolicit_ctx_class(self, proxy): proxy = BoundLoggerLazyProxy(None, initial_values={'a': 1, 'b': 2}) b = proxy.new(foo=42) assert {'a': 1, 'b': 2, 'foo': 42} == b._context
def test_unbind_unbinds_from_initial_values(self): p = BoundLoggerLazyProxy(None, initial_values={'a': 1, 'b': 2}) b = p.unbind('a') assert {'b': 2} == b._context
def test_bind_honors_initial_values(self): p = BoundLoggerLazyProxy(None, initial_values={'a': 1, 'b': 2}) b = p.bind() assert {'a': 1, 'b': 2} == b._context b = p.bind(c=3) assert {'a': 1, 'b': 2, 'c': 3} == b._context
def proxy(): """ Returns a BoundLoggerLazyProxy constructed w/o paramaters & None as logger. """ return BoundLoggerLazyProxy(None)