def log_sql_targets(conn_str, path, sql_query, is_succeed):
try:
for target_op in extract_from_sql(path, sql_query):
target_op = attr.evolve(target_op, success=is_succeed)
if (target_op.path.startswith("snowflake")
and TrackingConfig.from_databand_context(
).value_reporting_strategy == ValueTrackingLevel.ALL
and not target_op.name.startswith("@")):
# 1) snowflake tables are lazy evaluated types
# we support they been logs only if the the strategy is All
# 2) staging is not supported yet
log_snowflake_table_targets(table_op=target_op,
connection_string=conn_str)
else:
# the target can actually be a non table like S3 file that used
# as part of the sql query
log_dataset_op(
op_path=target_op.path,
op_type=target_op.operation,
success=target_op.success,
)
except Exception:
logger.exception(
"Caught exception will trying to log target for sql_query")
def task_log_dataset_op_nested_json_data():
log_dataset_op(
op_path="/my/path/to/nested_data.json",
op_type=DbndDatasetOperationType.write,
data=nested_json,
with_histograms=with_histograms,
with_stats=with_stats,
)
def task_log_dataset_op_nested_json_data():
log_dataset_op(
op_path="/my/path/to/nested_data.json",
op_type=DbndDatasetOperationType.read,
data=nested_json,
with_schema=schema,
with_preview=preview,
with_histograms=True,
with_partition=False,
)
def log_snowflake_table_targets(
table_op: TableTargetOperation,
connection_string: Union[str, SnowflakeConnection],
with_preview: Optional[bool] = None,
with_schema: Optional[bool] = None,
):
if not is_plugin_enabled("dbnd-snowflake", module_import="dbnd_snowflake"):
return
from dbnd_snowflake.snowflake_values import SnowflakeTable
with SnowflakeController(connection_string) as snowflake_ctrl:
snowflake_table = SnowflakeTable.from_table(snowflake_ctrl,
table_op.name)
log_dataset_op(
op_path=table_op.path,
op_type=table_op.operation,
success=table_op.success,
data=snowflake_table,
with_preview=with_preview,
with_schema=with_schema,
)