def from_json(cls, event: str) -> HeucodEvent:
if not event:
raise ValueError("The string can't be empty or None")
try:
json_obj = json.loads(event)
except json.JSONDecodeError as ex:
raise ex from None
instance = cls()
# Convert the names of the JSON attributes to snake case (from camel case).
obj_dict = {}
for k, v in json_obj.items():
if k != "id":
key_tokens = re.split("(?=[A-Z])", k)
obj_dict["_".join([t.lower() for t in key_tokens])] = v
else:
# The id_ attribtues is an exception of the naming standard. In Python, id is a
# reserved word and its use for naming variables/attribtues/... should be avoided.
# Thus the name id_.
obj_dict["id_"] = v
instance = dataclass_replace(instance, **obj_dict)
return instance
def squashed(self) -> Iterator[Tuple[StateTransition, Timedelta]]:
"""Returns an iterator over the latest state transitions, together with
the total duration of the transitioned-from state.
The iterator yields transitions *newest to oldest*, not the other way around.
Any transitions that did not actually change the state (i.e. where
transitioned-from and transitioned-to state are the same) are squashed,
their duration is added to the total duration.
An empty history returns an empty iterator.
A StateTransitionHistory does not squash states which do change state on insertion.
Use this method if you want to answer questions like "How long has this
metric been in the current state?" and "What was the the previous
state, diffent from the current one, and how long did it last?":
>>> # Answer the first question:
>>> (current_state, duration) = next(history.squashed())
(State.WARNING, <Timedelta ...>)
>>> # ...and the second question:
>>> (previous_state, duration) = next(next(history.squashed()))
(State.OK, <Timedelta ...>)
"""
if self.is_empty():
return
candidate_transition = dataclass_replace(self._transitions[-1])
# Iterate over all transitions, in reverse order, starting with the second to last.
# The `candidate_transition` marks the last transition in a chain of transitions
# `S -> S -> ... -> S` for some state `S`. We aim to squash this chain.
# Skip a transition preceeding the candidate as long as it did not change state.
# If it did, compute the total duration that the candidate state lasted
# using the fact that state `S` was entered at `candidate.time` and
# left at `candidate_transition.time`:
#
# ┄┄┄┄┄┄╮┄┄┄┄┄┄┄╮
# │ │┄┄┄┄┄┄┄╮ ┄┄┄┄┄┄┄╮┄┄┄┄┄┄┄╮
# │ T │ S │ ⋯ S │ S │ ⋯
# ⋯──────┴───────┴───────┴─────────────┴───────┴──────→ time
# ↑ ↑
# transition.time candidate_transition.time
#
# After that, recorded the current transition as a new candidate, since
# it marks the time when the metric left some state; i.e. it is at the
# start of its own chain of transitions `T -> T -> ... -> T`.
for transition in self._transitions[-2::-1]:
if transition.state == candidate_transition.state:
continue
else:
yield (
candidate_transition,
candidate_transition.time - transition.time,
)
candidate_transition = transition
else:
yield (candidate_transition,
candidate_transition.time - self.epoch)
def set_name(self, name: str, value: "Value") -> "State":
old_scope: Scope = self.scopes[self.current_scope_id]
new_scope = old_scope.with_member(name, value)
return dataclass_replace(
self,
scopes=self.scopes.set(self.current_scope_id, new_scope)
)
def _replace_tr(ex_, active_list_):
data_ids = [-1] * len(ex_.full_sentences)
for i, t in enumerate(ex_.tr_target):
if t is not None:
# keep only the first target if there are multiple
data_ids[i + ex_.offset] = len(active_list_)
active_list_.append(t[0])
return dataclass_replace(ex_, tr_target=data_ids)
def make_scope(self, parent_id: int, new_id: int, persistent: bool = False) -> "State":
assert parent_id in self.scopes
assert new_id not in self.scopes
new_scope = Scope(parent_id, new_id, Map(), persistent=persistent)
return dataclass_replace(
self.set_scope(new_id, new_scope),
scope_stack=(new_id, self.scope_stack)
)
def test_new_source_column_and_force_rebuild(self):
# First run - should create tracking table, target table, and insert values
run_druzhba(self.args)
with self.source_conn.cursor() as cur:
cur.execute(
"ALTER TABLE test_basic ADD COLUMN new_value VARCHAR(63) DEFAULT 'default'"
)
cur.execute(
"INSERT INTO test_basic VALUES (%s, %s, %s, %s, %s, %s);",
(3, t.t2, "drop", 3, "c", "other"),
)
# Second run - should not fail despite the new column. Should pick up new row.
run_druzhba(self.args)
with self.target_conn.cursor() as cur:
cur.execute("SELECT COUNT(*) FROM druzhba_test.test_basic")
self.assertEqual(cur.fetchone()[0], 3)
# Third run - should recreate target table with the new column and fully refresh
# TODO: Check user permissions are maintained
run_druzhba(dataclass_replace(self.args, rebuild=True))
with self.target_conn.cursor() as cur:
cur.execute("SELECT * FROM druzhba_test.test_basic ORDER BY pk")
results = cur.fetchall()
self.assertListEqual(
results,
[
(1, t.t0, 0, "a", "default"),
(2, t.t1, 1, "b", "default"),
(3, t.t2, 3, "c", "other"),
],
)
cur.execute(
"SELECT * FROM druzhba_test.pipeline_table_index ORDER BY created_ts"
)
results = cur.fetchall()
self.assertListEqual(
results,
[
(
"pgtest",
"druzhba_test",
"test_basic",
t.t1.strftime("%Y-%m-%d %H:%M:%S.%f"),
ANY,
),
(
"pgtest",
"druzhba_test",
"test_basic",
t.t2.strftime("%Y-%m-%d %H:%M:%S.%f"),
ANY,
),
(
"pgtest",
"druzhba_test",
"test_basic",
t.t2.strftime("%Y-%m-%d %H:%M:%S.%f"),
ANY,
),
],
)
def test_force_refresh(self):
# First run - should create tracking table, target table, and insert values
run_druzhba(self.args)
with self.source_conn.cursor() as cur:
cur.execute(
"INSERT INTO test_basic VALUES (%s, %s, %s, %s, %s);",
(3, t.t2, "drop", 3, "c"),
)
cur.execute("DELETE FROM test_basic WHERE pk = %s", (1, ))
# Second run - should pick up delete and insert
run_druzhba(dataclass_replace(self.args, full_refresh=True))
with self.target_conn.cursor() as cur:
cur.execute("SELECT * FROM druzhba_test.test_basic ORDER BY pk")
results = cur.fetchall()
self.assertListEqual(results, [(2, t.t1, 1, "b"),
(3, t.t2, 3, "c")])
cur.execute(
"SELECT * FROM druzhba_test.pipeline_table_index ORDER BY created_ts"
)
results = cur.fetchall()
self.assertListEqual(
results,
[
(
"pgtest",
"druzhba_test",
"test_basic",
t.t1.strftime("%Y-%m-%d %H:%M:%S.%f"),
ANY,
),
(
"pgtest",
"druzhba_test",
"test_basic",
t.t2.strftime("%Y-%m-%d %H:%M:%S.%f"),
ANY,
),
],
)
with self.source_conn.cursor() as cur:
cur.execute(
"INSERT INTO test_basic VALUES (%s, %s, %s, %s, %s);",
(1, t.t0, "value", 0, "a"),
)
cur.execute(
"UPDATE test_basic SET value1 = 2, updated_at = %s WHERE pk = 2",
(t.t3, ),
)
# Third run: incremental updates should proceed from second run index value,
# so re-insert of value 1 at original `updated_at` is not picked up while new update is.
run_druzhba(self.args)
with self.target_conn.cursor() as cur:
cur.execute("SELECT * FROM druzhba_test.test_basic ORDER BY pk")
results = cur.fetchall()
self.assertListEqual(results, [(2, t.t3, 2, "b"),
(3, t.t2, 3, "c")])
cur.execute(
"SELECT * FROM druzhba_test.pipeline_table_index ORDER BY created_ts"
)
results = cur.fetchall()
self.assertListEqual(
results,
[
(
"pgtest",
"druzhba_test",
"test_basic",
t.t1.strftime("%Y-%m-%d %H:%M:%S.%f"),
ANY,
),
(
"pgtest",
"druzhba_test",
"test_basic",
t.t2.strftime("%Y-%m-%d %H:%M:%S.%f"),
ANY,
),
(
"pgtest",
"druzhba_test",
"test_basic",
t.t3.strftime("%Y-%m-%d %H:%M:%S.%f"),
ANY,
),
],
)
def _with_boxes(self, boxes: Map):
return dataclass_replace(self, boxes=boxes)
def with_df(self, df):
return dataclass_replace(self, df=df)
def set_scope(self, id: int, scope: Scope) -> "State":
return dataclass_replace(self, scopes=self.scopes.set(id, scope))
def pop_scope(self) -> "State":
return dataclass_replace(
self,
scope_stack=self.scope_stack[1]
)
def increment_box_id(self):
return dataclass_replace(self, last_box_id=self.last_box_id + 1)
def _with_boxes_in_transaction(self, in_transaction: Map):
return dataclass_replace(self, box_in_transaction=in_transaction)
def with_filename(self, filename):
return dataclass_replace(self, filename=filename)
def kill_scope(self, scope_id: int) -> "State":
return dataclass_replace(
self,
scopes=self.scopes.delete(scope_id)
)
def with_status(self, status):
"""Sets the status in a copy of the report and returns it.
"""
return dataclass_replace(self, status=status)
def with_name(self, name: str) -> Code:
return dataclass_replace(self, name=name)
def deserialize(serialized_obj) -> 'UserInfo':
new_obj = UserInfo(id=-1, )
return dataclass_replace(new_obj, **serialized_obj)
def with_stack(self, stack: Stack):
return dataclass_replace(self, stack=stack)