def test_field_things():
dict_snap = snap_from_dagster_type({
"req":
int,
"opt":
Field(int, is_required=False),
"opt_with_default":
Field(int, is_required=False, default_value=2),
"req_with_desc":
Field(int, description="A desc"),
})
assert dict_snap.fields and len(dict_snap.fields) == 4
field_snap_dict = {
field_snap.name: field_snap
for field_snap in dict_snap.fields
}
assert field_snap_dict["req"].is_required is True
assert field_snap_dict["req"].description is None
assert field_snap_dict["opt"].is_required is False
assert field_snap_dict["opt"].default_provided is False
assert field_snap_dict["opt"].default_value_as_json_str is None
assert field_snap_dict["opt_with_default"].is_required is False
assert field_snap_dict["opt_with_default"].default_provided is True
assert deserialize_value(
field_snap_dict["opt_with_default"].default_value_as_json_str) == 2
assert field_snap_dict["req_with_desc"].is_required is True
assert field_snap_dict["req_with_desc"].description == "A desc"
def test_dead_events():
snapshot = path.join(path.dirname(path.realpath(__file__)),
'dead_events.txt')
with open(snapshot, 'r') as fd:
objs = []
for line in fd.readlines():
obj = deserialize_value(line)
assert obj is not None
objs.append(obj)
assert len(objs) == 6
def _construct_fields(config_type_snap, config_snap_map):
return {
field.name: Field(
construct_config_type_from_snap(config_snap_map[field.type_key],
config_snap_map),
description=field.description,
is_required=field.is_required,
default_value=deserialize_value(field.default_value_as_json_str)
if field.default_provided else FIELD_NO_DEFAULT_PROVIDED,
)
for field in config_type_snap.fields
}
def _construct_fields(
config_type_snap: ConfigTypeSnap,
config_snap_map: Dict[str, ConfigTypeSnap],
) -> Dict[str, Field]:
fields = check.not_none(config_type_snap.fields)
return {
cast(str, field.name): Field(
construct_config_type_from_snap(config_snap_map[field.type_key],
config_snap_map),
description=field.description,
is_required=field.is_required,
default_value=deserialize_value(
cast(str, field.default_value_as_json_str))
if field.default_provided else FIELD_NO_DEFAULT_PROVIDED,
)
for field in fields
}
def launch_step(
self,
step_context: StepExecutionContext,
prior_attempts_count: int,
) -> Iterator[DagsterEvent]:
step_run_ref = step_context_to_step_run_ref(step_context,
prior_attempts_count)
run_id = step_context.pipeline_run.run_id
step_run_dir = os.path.join(self.scratch_dir, run_id,
step_run_ref.step_key)
os.makedirs(step_run_dir)
step_run_ref_file_path = os.path.join(step_run_dir,
PICKLED_STEP_RUN_REF_FILE_NAME)
with open(step_run_ref_file_path, "wb") as step_pickle_file:
pickle.dump(step_run_ref, step_pickle_file)
command_tokens = [
sys.executable,
"-m",
"dagster.core.execution.plan.local_external_step_main",
step_run_ref_file_path,
]
# If this is being called within a `capture_interrupts` context, allow interrupts
# while waiting for the subprocess to complete, so that we can terminate slow or
# hanging steps
with raise_execution_interrupts():
subprocess.call(command_tokens,
stdout=sys.stdout,
stderr=sys.stderr)
events_file_path = os.path.join(step_run_dir, PICKLED_EVENTS_FILE_NAME)
file_manager = LocalFileManager(".")
events_file_handle = LocalFileHandle(events_file_path)
events_data = file_manager.read_data(events_file_handle)
all_events = deserialize_value(pickle.loads(events_data))
for event in all_events:
# write each pickled event from the external instance to the local instance
step_context.instance.handle_new_event(event)
if event.is_dagster_event:
yield event.dagster_event
def _get_step_records():
serialized_records = self.databricks_runner.client.read_file(path)
if not serialized_records:
return []
return deserialize_value(pickle.loads(serialized_records))
def test_deserialize_empty_set():
assert set() == deserialize_value(serialize_value(set()))
assert frozenset() == deserialize_value(serialize_value(frozenset()))
def test_deserialize_value_ok():
unpacked_tuple = deserialize_value('{"foo": "bar"}')
assert unpacked_tuple
assert unpacked_tuple["foo"] == "bar"
def get_step_events(self, run_id, step_key):
path = self._dbfs_path(run_id, step_key, PICKLED_EVENTS_FILE_NAME)
events_data = self.databricks_runner.client.read_file(path)
return deserialize_value(pickle.loads(events_data))
