import json
from typing import Optional
from contextlib import contextmanager
import grpc
from google.protobuf.empty_pb2 import Empty
from cicada2.shared.logs import get_logger
from cicada2.protos import runner_pb2, runner_pb2_grpc
from cicada2.shared.types import ActionResult, AssertResult
LOGGER = get_logger("messaging")
# NOTE: stream action requests and results for multiple executions?
# NOTE: support for non-json types with encoding param?
@contextmanager
def get_action_sender(runner_address: str) -> Optional[ActionResult]:
with grpc.insecure_channel(runner_address) as channel:
stub = runner_pb2_grpc.RunnerStub(channel)
def call(action: dict):
request = runner_pb2.ActionRequest(type=action["type"],
params=json.dumps(
action["params"]))
try:
response: runner_pb2.ActionReply = stub.Action(request)
return json.loads(response.outputs)
except json.JSONDecodeError as err:
from cicada2.engine.config import (
CONTAINER_NETWORK,
CREATE_NETWORK,
HEALTHCHECK_INITIAL_WAIT,
HEALTHCHECK_MAX_RETRIES,
POD_NAMESPACE,
POD_SERVICE_ACCOUNT,
)
from cicada2.shared.errors import ValidationError
from cicada2.shared.logs import get_logger
from cicada2.engine.messaging import runner_healthcheck
from cicada2.engine.parsing import render_section
from cicada2.engine.testing import run_test_with_timeout
from cicada2.shared.types import TestConfig, RunnerClosure, TestSummary, Volume
LOGGER = get_logger("runners")
def runner_to_image(runner_name: str) -> Optional[str]:
"""
Determine docker image based on runner name
Args:
runner_name: Type of test runner
Returns:
Docker image for runner
"""
if runner_name == "rest-runner":
return "cicadatesting/cicada-2-rest-runner"
elif runner_name == "sql-runner":
from os import getenv
from contextlib import contextmanager
from datetime import datetime
from typing import Dict, List, Optional
from typing_extensions import TypedDict
from kafka import KafkaConsumer, KafkaProducer
from kafka.errors import KafkaError
from cicada2.shared.asserts import assert_dicts
from cicada2.shared.types import AssertResult
from cicada2.shared.logs import get_logger
from cicada2.shared.util import get_runtime_ms
LOGGER = get_logger("kafka-runner")
class KafkaMessage(TypedDict):
topic: Optional[str]
key: Optional[str]
value: str
class ActionParams(TypedDict):
topic: str
timeout_ms: Optional[int]
max_records: Optional[int]
key: Optional[str]
messages: Optional[List[KafkaMessage]]
offset: Optional[str]
from cicada2.engine.actions import run_actions, combine_action_data
from cicada2.engine.asserts import get_remaining_asserts, run_asserts
from cicada2.shared.logs import get_logger
from cicada2.engine.state import combine_data_by_key, create_item_name
from cicada2.shared.types import (
Action,
ActionsData,
Assert,
Statuses,
TestConfig,
TestSummary,
)
LOGGER = get_logger("testing")
def get_default_cycles(actions: List[Action], asserts: List[Assert]) -> int:
"""
Determine number of default cycles for test given actions and asserts in it
* If there are asserts, by default run unlimited
* If there are no asserts but actions, run only once
* Otherwise, the test has no actions or asserts so do not run
Args:
actions: list of actions in test
asserts: list of asserts in test
Returns:
# Determine if all tests passed or failed
# exit code accordingly (with logging)
from urllib.parse import urlparse
from os import getenv
from datetime import datetime, timedelta
import sys
import json
import time
from s3fs import S3FileSystem
import click
from cicada2.engine.reporting import test_succeeded
from cicada2.shared.logs import get_logger
LOGGER = get_logger("verification")
def s3fs_client() -> S3FileSystem:
return S3FileSystem(
key=getenv("S3_ACCESS_KEY_ID"),
secret=getenv("S3_SECRET_ACCESS_KEY"),
token=getenv("S3_SESSION_TOKEN"),
use_ssl=getenv("USE_SSL", "true").lower() in ["true", "t", "y", "yes"],
client_kwargs={
"endpoint_url": getenv("S3_ENDPOINT"),
"region_name": getenv("S3_REGION"),
},
skip_instance_cache=True,
)
from typing import Dict, List, Optional
from dask.distributed import Client, Future
from cicada2.engine.config import (
INITIAL_STATE_FILE,
TASK_TYPE,
REPORTS_FOLDER,
TESTS_FOLDER,
)
from cicada2.engine.loading import load_tests_tree
from cicada2.shared.logs import get_logger
from cicada2.engine.reporting import render_report
from cicada2.shared.types import TestSummary
LOGGER = get_logger("scheduling")
def sort_dependencies(dependency_map: Dict[str, List[str]]) -> List[str]:
added_names = set()
sorted_names = []
def add_name(name):
for dependency_name in dependency_map[name]:
add_name(dependency_name)
if name not in added_names:
added_names.add(name)
sorted_names.append(name)
for test_name in dependency_map:
import json
import base64
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
import grpc
# from grpc_status import rpc_status
from google.protobuf.json_format import Parse
from google.protobuf.json_format import MessageToDict
from typing_extensions import TypedDict
from cicada2.shared.asserts import assert_element
from cicada2.shared.logs import get_logger
from cicada2.shared.types import AssertResult
LOGGER = get_logger("grpc-runner")
class Metadata(TypedDict):
key: str
value: str
class ActionParams(TypedDict):
proto: str
service: str
serviceAddress: str
message: Optional[dict]
messages: Optional[List[dict]]
metadata: Optional[List[Metadata]]
compression: Optional[str]