def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
c.silent = True
parser.add_argument("--scenario",
metavar="SCENARIO",
type=str,
help="Scenario to run.")
parser.add_argument("--check",
metavar="CHECK",
type=str,
help="Check to run.")
args = parser.parse_args()
c.up("testdrive", persistent=True)
# c.start_and_wait_for_tcp(
# services=["zookeeper", "kafka", "schema-registry", "postgres"]
# )
scenarios = ([globals()[args.scenario]]
if args.scenario else Scenario.__subclasses__())
checks = [globals()[args.check]] if args.check else Check.__subclasses__()
for scenario_class in scenarios:
print(f"Testing upgrade scenario {scenario_class}")
scenario = scenario_class(checks=checks)
scenario.run(c)
def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
"""Feature benchmark framework."""
c.silent = True
parser.add_argument(
"--redpanda",
action="store_true",
help="run against Redpanda instead of the Confluent Platform",
)
parser.add_argument(
"--this-tag",
metavar="TAG",
type=str,
default=os.getenv("THIS_TAG", None),
help=
"'This' Materialize container tag to benchmark. If not provided, the current source will be used.",
)
parser.add_argument(
"--this-options",
metavar="OPTIONS",
type=str,
default=os.getenv("THIS_OPTIONS", None),
help="Options to pass to the 'This' instance.",
)
parser.add_argument(
"--other-tag",
metavar="TAG",
type=str,
default=os.getenv("OTHER_TAG", None),
help=
"'Other' Materialize container tag to benchmark. If not provided, the current source will be used.",
)
parser.add_argument(
"--other-options",
metavar="OPTIONS",
type=str,
default=os.getenv("OTHER_OPTIONS", None),
help="Options to pass to the 'Other' instance.",
)
parser.add_argument(
"--root-scenario",
"--scenario",
metavar="SCENARIO",
type=str,
default="Scenario",
help=
"Scenario or scenario family to benchmark. See scenarios.py for available scenarios.",
)
parser.add_argument(
"--scale",
metavar="+N | -N | N",
type=str,
default=None,
help="Absolute or relative scale to apply.",
)
parser.add_argument(
"--max-measurements",
metavar="N",
type=int,
default=99,
help="Limit the number of measurements to N.",
)
parser.add_argument(
"--max-retries",
metavar="N",
type=int,
default=3,
help="Retry any potential performance regressions up to N times.",
)
parser.add_argument(
"--this-nodes",
metavar="N",
type=int,
default=None,
help="Start a cluster with that many nodes for 'THIS'",
)
parser.add_argument(
"--other-nodes",
metavar="N",
type=int,
default=None,
help="Start a cluster with that many nodes for 'OTHER'",
)
parser.add_argument(
"--this-workers",
metavar="N",
type=int,
default=None,
help="Number of workers to use for 'THIS'",
)
parser.add_argument(
"--other-workers",
metavar="N",
type=int,
default=None,
help="Number of workers to use for 'OTHER'",
)
args = parser.parse_args()
print(f"""
this_tag: {args.this_tag}
this_options: {args.this_options}
other_tag: {args.other_tag}
other_options: {args.other_options}
root_scenario: {args.root_scenario}""")
# Build the list of scenarios to run
root_scenario = globals()[args.root_scenario]
initial_scenarios = {}
if root_scenario.__subclasses__():
for scenario in root_scenario.__subclasses__():
has_children = False
for s in scenario.__subclasses__():
has_children = True
initial_scenarios[s] = 1
if not has_children:
initial_scenarios[scenario] = 1
else:
initial_scenarios[root_scenario] = 1
dependencies = ["postgres"]
if args.redpanda:
dependencies += ["redpanda"]
else:
dependencies += ["zookeeper", "kafka", "schema-registry"]
c.start_and_wait_for_tcp(services=dependencies)
scenarios = initial_scenarios.copy()
for cycle in range(0, args.max_retries):
print(
f"Cycle {cycle+1} with scenarios: {', '.join([scenario.__name__ for scenario in scenarios.keys()])}"
)
report = Report()
for scenario in list(scenarios.keys()):
comparison = run_one_scenario(c, scenario, args)
report.append(comparison)
if not comparison.is_regression():
del scenarios[scenario]
print(f"+++ Benchmark Report for cycle {cycle+1}:")
report.dump()
if len(scenarios.keys()) == 0:
break
if len(scenarios.keys()) > 0:
print(
f"ERROR: The following scenarios have regressions: {', '.join([scenario.__name__ for scenario in scenarios.keys()])}"
)
sys.exit(1)
def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
"""Feature benchmark framework."""
c.silent = True
parser.add_argument(
"--this-tag",
metavar="TAG",
type=str,
default=os.getenv("THIS_TAG", None),
help="'This' Materialize container tag to benchmark. If not provided, the current source will be used.",
)
parser.add_argument(
"--this-options",
metavar="OPTIONS",
type=str,
default=os.getenv("THIS_OPTIONS", None),
help="Options to pass to the 'This' instance.",
)
parser.add_argument(
"--other-tag",
metavar="TAG",
type=str,
default=os.getenv("OTHER_TAG", None),
help="'Other' Materialize container tag to benchmark. If not provided, the current source will be used.",
)
parser.add_argument(
"--other-options",
metavar="OPTIONS",
type=str,
default=os.getenv("OTHER_OPTIONS", None),
help="Options to pass to the 'Other' instance.",
)
parser.add_argument(
"--root-scenario",
"--scenario",
metavar="SCENARIO",
type=str,
default="Scenario",
help="Scenario or scenario family to benchmark. See scenarios.py for available scenarios.",
)
parser.add_argument(
"--scale",
metavar="+N | -N | N",
type=str,
default=None,
help="Absolute or relative scale to apply.",
)
parser.add_argument(
"--max-runs",
metavar="N",
type=int,
default=99,
help="Limit the number of executions to N.",
)
args = parser.parse_args()
print(
f"""
this_tag: {args.this_tag}
this_options: {args.this_options}
other_tag: {args.other_tag}
other_options: {args.other_options}
root_scenario: {args.root_scenario}"""
)
# Build the list of scenarios to run
root_scenario = globals()[args.root_scenario]
scenarios = []
if root_scenario.__subclasses__():
for scenario in root_scenario.__subclasses__():
has_children = False
for s in scenario.__subclasses__():
has_children = True
scenarios.append(s)
if not has_children:
scenarios.append(scenario)
else:
scenarios.append(root_scenario)
print(f"scenarios: {', '.join([scenario.__name__ for scenario in scenarios])}")
c.start_and_wait_for_tcp(
services=["zookeeper", "kafka", "schema-registry", "postgres"]
)
report = Report()
has_regressions = False
for scenario in scenarios:
comparison = run_one_scenario(c, scenario, args)
report.append(comparison)
if comparison.is_regression():
has_regressions = True
report.dump()
print("+++ Benchmark Report")
report.dump()
if has_regressions:
print("ERROR: benchmarks have regressions")
sys.exit(1)
