def ExternalPartitionConfig(self, request, _context):
partition_args = deserialize_json_to_dagster_namedtuple(
request.serialized_partition_args)
check.inst_param(partition_args, "partition_args", PartitionArgs)
recon_repo = self._recon_repository_from_origin(
partition_args.repository_origin)
definition = recon_repo.get_definition()
partition_set_def = definition.get_partition_set_def(
partition_args.partition_set_name)
partition = partition_set_def.get_partition(
partition_args.partition_name)
try:
with user_code_error_boundary(
PartitionExecutionError,
lambda:
"Error occurred during the evaluation of the `run_config_for_partition` "
"function for partition set {partition_set_name}".format(
partition_set_name=partition_set_def.name),
):
run_config = partition_set_def.run_config_for_partition(
partition)
return api_pb2.ExternalPartitionConfigReply(
serialized_external_partition_config_or_external_partition_execution_error
=serialize_dagster_namedtuple(
ExternalPartitionConfigData(name=partition.name,
run_config=run_config)))
except PartitionExecutionError:
return api_pb2.ExternalPartitionConfigReply(
serialized_external_partition_config_or_external_partition_execution_error
=serialize_dagster_namedtuple(
ExternalPartitionExecutionErrorData(
serializable_error_info_from_exc_info(
sys.exc_info()))))
def ExternalPartitionTags(self, request, _context):
partition_args = deserialize_json_to_dagster_namedtuple(
request.serialized_partition_args)
check.inst_param(partition_args, 'partition_args', PartitionArgs)
recon_repo = recon_repository_from_origin(
partition_args.repository_origin)
definition = recon_repo.get_definition()
partition_set_def = definition.get_partition_set_def(
partition_args.partition_set_name)
partition = partition_set_def.get_partition(
partition_args.partition_name)
try:
with user_code_error_boundary(
PartitionExecutionError,
lambda:
'Error occurred during the evaluation of the `tags_for_partition` function for '
'partition set {partition_set_name}'.format(
partition_set_name=partition_set_def.name),
):
tags = partition_set_def.tags_for_partition(partition)
return api_pb2.ExternalPartitionTagsReply(
serialized_external_partition_tags_or_external_partition_execution_error
=serialize_dagster_namedtuple(
ExternalPartitionTagsData(name=partition.name,
tags=tags)))
except PartitionExecutionError:
return api_pb2.ExternalPartitionTagsReply(
serialized_external_partition_tags_or_external_partition_execution_error
=serialize_dagster_namedtuple(
ExternalPartitionExecutionErrorData(
serializable_error_info_from_exc_info(
sys.exc_info()))))
def get_partition_set_execution_param_data(args):
check.inst_param(args, "args", PartitionSetExecutionParamArgs)
recon_repo = recon_repository_from_origin(args.repository_origin)
repo_definition = recon_repo.get_definition()
partition_set_def = repo_definition.get_partition_set_def(
args.partition_set_name)
try:
with user_code_error_boundary(
PartitionExecutionError,
lambda:
"Error occurred during the partition generation for partition set "
"{partition_set_name}".format(partition_set_name=
partition_set_def.name),
):
all_partitions = partition_set_def.get_partitions()
partitions = [
partition for partition in all_partitions
if partition.name in args.partition_names
]
partition_data = []
for partition in partitions:
def _error_message_fn(partition_set_name, partition_name):
return lambda: (
"Error occurred during the partition config and tag generation for "
"partition set {partition_set_name}::{partition_name}".
format(partition_set_name=partition_set_name,
partition_name=partition_name))
with user_code_error_boundary(
PartitionExecutionError,
_error_message_fn(partition_set_def.name, partition.name)):
run_config = partition_set_def.run_config_for_partition(
partition)
tags = partition_set_def.tags_for_partition(partition)
partition_data.append(
ExternalPartitionExecutionParamData(
name=partition.name,
tags=tags,
run_config=run_config,
))
return ExternalPartitionSetExecutionParamData(
partition_data=partition_data)
except PartitionExecutionError:
return ExternalPartitionExecutionErrorData(
serializable_error_info_from_exc_info(sys.exc_info()))
def get_partition_names(recon_repo, partition_set_name):
definition = recon_repo.get_definition()
partition_set_def = definition.get_partition_set_def(partition_set_name)
try:
with user_code_error_boundary(
PartitionExecutionError,
lambda:
f"Error occurred during the execution of the partition generation function for "
f"{_get_target_for_partition_execution_error(partition_set_def)}",
):
return ExternalPartitionNamesData(
partition_names=partition_set_def.get_partition_names())
except PartitionExecutionError:
return ExternalPartitionExecutionErrorData(
serializable_error_info_from_exc_info(sys.exc_info()))
def get_partition_tags(recon_repo, partition_set_name, partition_name):
definition = recon_repo.get_definition()
partition_set_def = definition.get_partition_set_def(partition_set_name)
partition = partition_set_def.get_partition(partition_name)
try:
with user_code_error_boundary(
PartitionExecutionError,
lambda:
"Error occurred during the evaluation of the `tags_for_partition` function for "
f"{_get_target_for_partition_execution_error(partition_set_def)}",
):
tags = partition_set_def.tags_for_partition(partition)
return ExternalPartitionTagsData(name=partition.name, tags=tags)
except PartitionExecutionError:
return ExternalPartitionExecutionErrorData(
serializable_error_info_from_exc_info(sys.exc_info()))
def get_partition_set_execution_param_data(recon_repo, partition_set_name,
partition_names):
repo_definition = recon_repo.get_definition()
partition_set_def = repo_definition.get_partition_set_def(
partition_set_name)
try:
with user_code_error_boundary(
PartitionExecutionError,
lambda: "Error occurred during the partition generation for "
f"{_get_target_for_partition_execution_error(partition_set_def)}",
):
all_partitions = partition_set_def.get_partitions()
partitions = [
partition for partition in all_partitions
if partition.name in partition_names
]
partition_data = []
for partition in partitions:
def _error_message_fn(partition_name):
return lambda: (
"Error occurred during the partition config and tag generation for "
f"'{partition_name}' in {_get_target_for_partition_execution_error(partition_set_def)}"
)
with user_code_error_boundary(PartitionExecutionError,
_error_message_fn(partition.name)):
run_config = partition_set_def.run_config_for_partition(
partition)
tags = partition_set_def.tags_for_partition(partition)
partition_data.append(
ExternalPartitionExecutionParamData(
name=partition.name,
tags=tags,
run_config=run_config,
))
return ExternalPartitionSetExecutionParamData(
partition_data=partition_data)
except PartitionExecutionError:
return ExternalPartitionExecutionErrorData(
serializable_error_info_from_exc_info(sys.exc_info()))
def get_partition_config(recon_repo, partition_set_name, partition_name):
definition = recon_repo.get_definition()
partition_set_def = definition.get_partition_set_def(partition_set_name)
partition = partition_set_def.get_partition(partition_name)
try:
with user_code_error_boundary(
PartitionExecutionError,
lambda:
"Error occurred during the evaluation of the `run_config_for_partition` "
"function for partition set {partition_set_name}".format(
partition_set_name=partition_set_def.name),
):
run_config = partition_set_def.run_config_for_partition(partition)
return ExternalPartitionConfigData(name=partition.name,
run_config=run_config)
except PartitionExecutionError:
return ExternalPartitionExecutionErrorData(
serializable_error_info_from_exc_info(sys.exc_info()))
def get_partition_names(args):
check.inst_param(args, 'args', PartitionNamesArgs)
recon_repo = recon_repository_from_origin(args.repository_origin)
definition = recon_repo.get_definition()
partition_set_def = definition.get_partition_set_def(args.partition_set_name)
try:
with user_code_error_boundary(
PartitionExecutionError,
lambda: 'Error occurred during the execution of the partition generation function for '
'partition set {partition_set_name}'.format(partition_set_name=partition_set_def.name),
):
return ExternalPartitionNamesData(
partition_names=partition_set_def.get_partition_names()
)
except PartitionExecutionError:
return ExternalPartitionExecutionErrorData(
serializable_error_info_from_exc_info(sys.exc_info())
)
def get_partition_tags(args):
check.inst_param(args, "args", PartitionArgs)
recon_repo = recon_repository_from_origin(args.repository_origin)
definition = recon_repo.get_definition()
partition_set_def = definition.get_partition_set_def(args.partition_set_name)
partition = partition_set_def.get_partition(args.partition_name)
try:
with user_code_error_boundary(
PartitionExecutionError,
lambda: "Error occurred during the evaluation of the `tags_for_partition` function for "
"partition set {partition_set_name}".format(partition_set_name=partition_set_def.name),
):
tags = partition_set_def.tags_for_partition(partition)
return ExternalPartitionTagsData(name=partition.name, tags=tags)
except PartitionExecutionError:
return ExternalPartitionExecutionErrorData(
serializable_error_info_from_exc_info(sys.exc_info())
)
def get_partition_config(args):
check.inst_param(args, 'args', PartitionArgs)
recon_repo = recon_repository_from_origin(args.repository_origin)
definition = recon_repo.get_definition()
partition_set_def = definition.get_partition_set_def(args.partition_set_name)
partition = partition_set_def.get_partition(args.partition_name)
try:
with user_code_error_boundary(
PartitionExecutionError,
lambda: 'Error occurred during the evaluation of the `run_config_for_partition` '
'function for partition set {partition_set_name}'.format(
partition_set_name=partition_set_def.name
),
):
run_config = partition_set_def.run_config_for_partition(partition)
return ExternalPartitionConfigData(name=partition.name, run_config=run_config)
except PartitionExecutionError:
return ExternalPartitionExecutionErrorData(
serializable_error_info_from_exc_info(sys.exc_info())
)
def ExternalPartitionSetExecutionParams(self, request, _context):
partition_set_execution_param_args = deserialize_json_to_dagster_namedtuple(
request.serialized_partition_set_execution_param_args)
check.inst_param(
partition_set_execution_param_args,
"partition_set_execution_param_args",
PartitionSetExecutionParamArgs,
)
recon_repo = self._recon_repository_from_origin(
partition_set_execution_param_args.repository_origin)
definition = recon_repo.get_definition()
partition_set_def = definition.get_partition_set_def(
partition_set_execution_param_args.partition_set_name)
try:
with user_code_error_boundary(
PartitionExecutionError,
lambda:
"Error occurred during the partition generation for partition set "
"{partition_set_name}".format(partition_set_name=
partition_set_def.name),
):
all_partitions = partition_set_def.get_partitions()
partitions = [
partition for partition in all_partitions if partition.name in
partition_set_execution_param_args.partition_names
]
partition_data = []
for partition in partitions:
def _error_message_fn(partition_set_name, partition_name):
return lambda: (
"Error occurred during the partition config and tag generation for "
"partition set {partition_set_name}::{partition_name}".
format(partition_set_name=partition_set_name,
partition_name=partition_name))
with user_code_error_boundary(
PartitionExecutionError,
_error_message_fn(partition_set_def.name,
partition.name),
):
run_config = partition_set_def.run_config_for_partition(
partition)
tags = partition_set_def.tags_for_partition(partition)
partition_data.append(
ExternalPartitionExecutionParamData(
name=partition.name,
tags=tags,
run_config=run_config,
))
return api_pb2.ExternalPartitionSetExecutionParamsReply(
serialized_external_partition_set_execution_param_data_or_external_partition_execution_error
=serialize_dagster_namedtuple(
ExternalPartitionSetExecutionParamData(
partition_data=partition_data)))
except PartitionExecutionError:
return api_pb2.ExternalPartitionSetExecutionParamsReply(
ExternalPartitionExecutionErrorData(
serializable_error_info_from_exc_info(sys.exc_info())))