def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None: c.silent = True parser.add_argument("--scenario", metavar="SCENARIO", type=str, help="Scenario to run.") parser.add_argument("--check", metavar="CHECK", type=str, help="Check to run.") args = parser.parse_args() c.up("testdrive", persistent=True) # c.start_and_wait_for_tcp( # services=["zookeeper", "kafka", "schema-registry", "postgres"] # ) scenarios = ([globals()[args.scenario]] if args.scenario else Scenario.__subclasses__()) checks = [globals()[args.check]] if args.check else Check.__subclasses__() for scenario_class in scenarios: print(f"Testing upgrade scenario {scenario_class}") scenario = scenario_class(checks=checks) scenario.run(c)
def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None: """Feature benchmark framework.""" c.silent = True parser.add_argument( "--redpanda", action="store_true", help="run against Redpanda instead of the Confluent Platform", ) parser.add_argument( "--this-tag", metavar="TAG", type=str, default=os.getenv("THIS_TAG", None), help= "'This' Materialize container tag to benchmark. If not provided, the current source will be used.", ) parser.add_argument( "--this-options", metavar="OPTIONS", type=str, default=os.getenv("THIS_OPTIONS", None), help="Options to pass to the 'This' instance.", ) parser.add_argument( "--other-tag", metavar="TAG", type=str, default=os.getenv("OTHER_TAG", None), help= "'Other' Materialize container tag to benchmark. If not provided, the current source will be used.", ) parser.add_argument( "--other-options", metavar="OPTIONS", type=str, default=os.getenv("OTHER_OPTIONS", None), help="Options to pass to the 'Other' instance.", ) parser.add_argument( "--root-scenario", "--scenario", metavar="SCENARIO", type=str, default="Scenario", help= "Scenario or scenario family to benchmark. See scenarios.py for available scenarios.", ) parser.add_argument( "--scale", metavar="+N | -N | N", type=str, default=None, help="Absolute or relative scale to apply.", ) parser.add_argument( "--max-measurements", metavar="N", type=int, default=99, help="Limit the number of measurements to N.", ) parser.add_argument( "--max-retries", metavar="N", type=int, default=3, help="Retry any potential performance regressions up to N times.", ) parser.add_argument( "--this-nodes", metavar="N", type=int, default=None, help="Start a cluster with that many nodes for 'THIS'", ) parser.add_argument( "--other-nodes", metavar="N", type=int, default=None, help="Start a cluster with that many nodes for 'OTHER'", ) parser.add_argument( "--this-workers", metavar="N", type=int, default=None, help="Number of workers to use for 'THIS'", ) parser.add_argument( "--other-workers", metavar="N", type=int, default=None, help="Number of workers to use for 'OTHER'", ) args = parser.parse_args() print(f""" this_tag: {args.this_tag} this_options: {args.this_options} other_tag: {args.other_tag} other_options: {args.other_options} root_scenario: {args.root_scenario}""") # Build the list of scenarios to run root_scenario = globals()[args.root_scenario] initial_scenarios = {} if root_scenario.__subclasses__(): for scenario in root_scenario.__subclasses__(): has_children = False for s in scenario.__subclasses__(): has_children = True initial_scenarios[s] = 1 if not has_children: initial_scenarios[scenario] = 1 else: initial_scenarios[root_scenario] = 1 dependencies = ["postgres"] if args.redpanda: dependencies += ["redpanda"] else: dependencies += ["zookeeper", "kafka", "schema-registry"] c.start_and_wait_for_tcp(services=dependencies) scenarios = initial_scenarios.copy() for cycle in range(0, args.max_retries): print( f"Cycle {cycle+1} with scenarios: {', '.join([scenario.__name__ for scenario in scenarios.keys()])}" ) report = Report() for scenario in list(scenarios.keys()): comparison = run_one_scenario(c, scenario, args) report.append(comparison) if not comparison.is_regression(): del scenarios[scenario] print(f"+++ Benchmark Report for cycle {cycle+1}:") report.dump() if len(scenarios.keys()) == 0: break if len(scenarios.keys()) > 0: print( f"ERROR: The following scenarios have regressions: {', '.join([scenario.__name__ for scenario in scenarios.keys()])}" ) sys.exit(1)
def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None: """Feature benchmark framework.""" c.silent = True parser.add_argument( "--this-tag", metavar="TAG", type=str, default=os.getenv("THIS_TAG", None), help="'This' Materialize container tag to benchmark. If not provided, the current source will be used.", ) parser.add_argument( "--this-options", metavar="OPTIONS", type=str, default=os.getenv("THIS_OPTIONS", None), help="Options to pass to the 'This' instance.", ) parser.add_argument( "--other-tag", metavar="TAG", type=str, default=os.getenv("OTHER_TAG", None), help="'Other' Materialize container tag to benchmark. If not provided, the current source will be used.", ) parser.add_argument( "--other-options", metavar="OPTIONS", type=str, default=os.getenv("OTHER_OPTIONS", None), help="Options to pass to the 'Other' instance.", ) parser.add_argument( "--root-scenario", "--scenario", metavar="SCENARIO", type=str, default="Scenario", help="Scenario or scenario family to benchmark. See scenarios.py for available scenarios.", ) parser.add_argument( "--scale", metavar="+N | -N | N", type=str, default=None, help="Absolute or relative scale to apply.", ) parser.add_argument( "--max-runs", metavar="N", type=int, default=99, help="Limit the number of executions to N.", ) args = parser.parse_args() print( f""" this_tag: {args.this_tag} this_options: {args.this_options} other_tag: {args.other_tag} other_options: {args.other_options} root_scenario: {args.root_scenario}""" ) # Build the list of scenarios to run root_scenario = globals()[args.root_scenario] scenarios = [] if root_scenario.__subclasses__(): for scenario in root_scenario.__subclasses__(): has_children = False for s in scenario.__subclasses__(): has_children = True scenarios.append(s) if not has_children: scenarios.append(scenario) else: scenarios.append(root_scenario) print(f"scenarios: {', '.join([scenario.__name__ for scenario in scenarios])}") c.start_and_wait_for_tcp( services=["zookeeper", "kafka", "schema-registry", "postgres"] ) report = Report() has_regressions = False for scenario in scenarios: comparison = run_one_scenario(c, scenario, args) report.append(comparison) if comparison.is_regression(): has_regressions = True report.dump() print("+++ Benchmark Report") report.dump() if has_regressions: print("ERROR: benchmarks have regressions") sys.exit(1)