def test_single_job_regressions(
descriptor_filename,
expected_yaml,
shared_datadir,
config_args,
config_env,
file_regression: FileRegressionFixture,
benchmark_event,
scripts,
parent_action_id,
):
random_object = random.Random()
random_object.seed(1)
if parent_action_id:
benchmark_event.parent_action_id = parent_action_id
expected_yaml = expected_yaml + f"-{parent_action_id}"
descriptor_data = toml.load(str(shared_datadir / descriptor_filename))
fetched_data_sources = generate_fetched_data_sources(descriptor_data)
fetched_models = generate_fetched_models(descriptor_data)
transpiler_config = create_executor_config(config_args, config_env)
yaml_spec = create_job_yaml_spec(
descriptor_data,
transpiler_config,
fetched_data_sources,
fetched_models,
scripts,
JOB_ID,
extra_bai_config_args=dict(random_object=random_object),
event=benchmark_event,
)
file_regression.check(yaml_spec, basename=expected_yaml, extension=".yaml")
def test_create_executor_config(config_args, config_env,
mock_availability_zones):
executor_config = create_executor_config(config_args, config_env)
args = get_args(config_args, config_env)
expected_descriptor_config = DescriptorConfig(
valid_strategies=args.valid_strategies,
valid_frameworks=args.valid_frameworks)
expected_bai_config = BaiConfig(
puller_docker_image=args.puller_docker_image,
puller_mount_chmod=args.puller_mount_chmod,
metrics_pusher_docker_image=args.metrics_pusher_docker_image,
metrics_extractor_docker_image=args.metrics_extractor_docker_image,
job_status_trigger_docker_image=args.job_status_trigger_docker_image,
cron_job_docker_image=args.cron_job_docker_image,
)
expected_environment_info = EnvironmentInfo(mock_availability_zones)
expected_executor_config = ExecutorConfig(
descriptor_config=expected_descriptor_config,
bai_config=expected_bai_config,
environment_info=expected_environment_info,
kubectl=args.kubectl,
)
assert executor_config == expected_executor_config
def main(argv=None):
common_kafka_cfg = get_kafka_service_config(SERVICE_NAME, argv)
executor_config = create_executor_config(argv, os.environ)
configure_logging(level=common_kafka_cfg.logging_level)
logger = logging.getLogger(SERVICE_NAME)
logger.info(f"Starting {SERVICE_NAME} Service: {SERVICE_DESCRIPTION}")
logger.info(f"common_args = {common_kafka_cfg}")
logger.info(f"executor_args = {executor_config}")
executor = create_executor(common_kafka_cfg, executor_config)
executor.run_loop()
def main(argv=None):
# This method is only called when using the transpiler as a module of its own, which
# is getting deprecated very soon
transpiler_config = create_executor_config(argv)
input = get_input_args(argv)
descriptor_data = toml.load(input.descriptor)
# TODO: Pass this as an argument
fetched_data_sources = descriptor_data.get("data", {}).get("sources", [])
yaml_string = create_job_yaml_spec(descriptor_data, transpiler_config,
fetched_data_sources, str(uuid4()))
if input.filename:
current_dir = os.path.dirname(os.path.abspath(__file__))
with open(os.path.join(current_dir, input.filename), "w") as f:
f.write(yaml_string)
else:
print(yaml_string)
def test_scheduled_job_regressions(
descriptor_filename,
expected_yaml,
shared_datadir,
config_args,
config_env,
file_regression: FileRegressionFixture,
benchmark_event,
):
random_object = random.Random()
random_object.seed(1)
descriptor_data = toml.load(str(shared_datadir / descriptor_filename))
transpiler_config = create_executor_config(config_args, config_env)
yaml_spec = create_scheduled_job_yaml_spec(descriptor_data,
transpiler_config,
JOB_ID,
event=benchmark_event)
file_regression.check(yaml_spec, basename=expected_yaml, extension=".yaml")
def test_raises_with_non_s3_puller_uri(shared_datadir, config_args, config_env,
file_regression: FileRegressionFixture,
benchmark_event):
random_object = random.Random()
random_object.seed(1)
descriptor_data = toml.load(str(shared_datadir / "training.toml"))
fetched_data_sources = generate_fetched_data_sources(descriptor_data,
http_puller_uri=True)
transpiler_config = create_executor_config(config_args, config_env)
with pytest.raises(DescriptorError):
create_job_yaml_spec(
descriptor_data,
transpiler_config,
fetched_data_sources,
[],
[],
JOB_ID,
extra_bai_config_args=dict(random_object=random_object),
event=benchmark_event,
)