def generate_sinusoids():
smm = SharedMemoryManager()
smm.start()
W = []
increment = 1.5 / (BITSTRING_SIZE + 1)
t = np.linspace(0, (4 * np.pi) / TARGET_W, N)
for i in range(BITSTRING_SIZE + 1):
if (0.5 + i * increment) == 1:
continue
w = TARGET_W / (0.5 + i * increment)
W.append(w)
SINUSOIDS.append(0.25 * np.cos(w * t))
TARGET = np.cos(TARGET_W * t)
w_list = ""
freqs = open("frequencies.ini", "w")
for i in range(BITSTRING_SIZE):
w_list += str(W[i]) + ","
w_list = w_list[0:len(w_list) - 2]
records = {"DEFAULT": {"frequencies": str(w_list), "target": TARGET_W}}
record = configparser.ConfigParser()
record.read_dict(records)
record.write(freqs)
freqs.close()
def main():
smm = SharedMemoryManager()
smm.start()
ls = smm.ShareableList(range(2000))
with Pool(4) as p:
print(*list(p.imap_unordered(f, ls)), sep='\n')
# print(p.map(f, [2, 3, 4, 5, 6])) # lock
# print(p.map(f, [2, 3, 4, 5, 6]))
smm.shutdown()
def start_shared_memory_manager():
# type: () -> SharedMemoryManager
""" Starts the shared memory manager.
:return: Shared memory manager instance.
"""
smm = SharedMemoryManager(address=('', PORT), authkey=AUTH_KEY)
smm.start()
return smm
def lif_feed_forward_benchmark(parameters: BenchmarkParameters):
shared = SharedMemoryManager()
shared.start()
params = list(parameters._asdict().values())
shared_list = shared.ShareableList(params)
run(["python3", __file__, shared_list.shm.name], stderr=STDOUT)
duration = shared_list[0]
shared_list.shm.close()
shared.shutdown()
return duration
def multiprocess(self, n, nthread=8):
"""
多进程 并发
:return:
"""
print('Parent process %s.' % os.getpid())
p = Pool(nthread) # 进程池, 和系统申请 nthread 个进程
smm = SharedMemoryManager() #TODO: pyrhon3.8+ 才有
smm.start() # Start the process that manages the shared memory blocks
cache_list = smm.ShareableList([0] * n)
# 限制了可被存储在其中的值只能是 int, float, bool, str （每条数据小于10M）, bytes （每条数据小于10M）以及 None 这些内置类型。
# 它另一个显著区别于内置 list 类型的地方在于它的长度无法修改（比如，没有 append, insert 等操作）
# 且不支持通过切片操作动态创建新的 ShareableList 实例。
shm_a = smm.SharedMemory(size=n)
# shm_a.buf[:] = bytearray([0]*n)
# shm_a.buf[:] = [0] * n
print('shm_a id in main process: {} '.format(id(shm_a)))
# 主进程的内存空间 和 子进程的内存空间 的考察
self.global_array = [0] * n
print('array id in main process: {} '.format(id(self.global_array)))
self.global_string = 'abc'
print('string id in main process: {} '.format(id(self.global_string)))
self.global_int = 10
print('int id in main process: {} '.format(id(self.global_int)))
for i in range(n):
# p.apply_async(task, args=(cache_name,i)) # apply_async 异步取回结果
p.apply_async(self.task, args=(cache_list, shm_a, i))
print('Waiting for all subprocesses done...')
p.close()
p.join()
print('All subprocesses done.')
smm.shutdown()
return cache_list, shm_a
def test_share_memory_rw():
cache_size = 10
init_frame = np.zeros((2160, 3840, 3), dtype=np.uint8)
global_index = Manager().Value('i', 1)
smm = SharedMemoryManager()
smm.start()
s1 = SharedMemoryFrameCache(smm, cache_size, init_frame.nbytes,
init_frame.shape)
with Pool(2) as pool:
r1 = pool.apply_async(blur, args=(
s1,
global_index,
'Shared Memory',
))
r2 = pool.apply_async(receive,
args=(
s1,
global_index,
'Shared Memory',
))
r1.get()
r2.get()
def main():
# This is the number of values that the writer will send to the reader
items_to_send = random.randint(10000, 100000)
smm = SharedMemoryManager()
smm.start()
# TODO - Create a ShareableList to be used between the processes
# TODO - Create any lock(s) or semaphore(s) that you feel you need
# TODO - create reader and writer processes
# TODO - Start the processes and wait for them to finish
print(f'{items_to_send} values sent')
# TODO - Display the number of numbers/items received by the reader.
# Can not use "items_to_send", must be a value collected
# by the reader processes.
# print(f'{<your variable>} values received')
smm.shutdown()
def main():
# This is the number of values that the writer will send to the reader
items_to_send = random.randint(100000, 1000000)
smm = SharedMemoryManager()
smm.start()
# Create a ShareableList to be used between the processes
size = BUFFER_SIZE + 2
shared_list = smm.ShareableList(range(size))
for i in range(0, size):
shared_list[i] = -1
# TODO - Create any lock(s) or semaphore(s) that you feel you need
lock = mp.Lock()
sem = mp.Semaphore(BUFFER_SIZE)
# TODO - create reader and writer processes
items_sent = 0
items_read = 0
p1 = mp.Process(target=write, args=(shared_list, items_sent, items_to_send, sem, lock))
p2 = mp.Process(target=read, args=(shared_list, items_read, items_to_send, sem, lock))
# TODO - Start the processes and wait for them to finish
p1.start()
p2.start()
p1.join()
p2.join()
print(f'{items_to_send} sent by the writer')
# TODO - Display the number of numbers/items received by the reader.
print(f'{shared_list[-1]} read by the reader')
smm.shutdown()
def parallel_eval_expr(n, ops, goal, timeout=1, proc_cnt=16):
q, q_type = make_tasks(n, ops)
task_cnt = len(q)
task_ranges = calc_task_ranges(proc_cnt, q)
smm = SharedMemoryManager()
smm.start()
task_mem, tq = make_shared_mem(smm, task_cnt, q_type)
tq[:] = q[:]
del q
res_mem, res = make_shared_mem(smm, task_cnt, np.int8)
res[:] = -2 # initial value indicating that result is not being calculated
start_time_mem, start_time = make_shared_mem(smm, proc_cnt, np.float_)
status_mem, status = make_shared_mem(smm, proc_cnt, np.int64)
status[:] = ProcStatus.IDLE
cur_idx_mem, cur_idx = make_shared_mem(smm, proc_cnt, np.int64)
cur_idx[:] = -1
processes = [None] * proc_cnt
restarts = [0] * proc_cnt
cur_t = time.time()
while not all(status[i] == ProcStatus.DONE for i in range(proc_cnt)):
print_status(res, status, cur_idx, restarts, proc_cnt)
try:
for i, p in enumerate(processes):
if p is None:
continue
if status[i] == ProcStatus.DONE:
continue
if (cur_t - start_time[i]) > timeout:
p.kill()
if cur_idx[i] < 0:
continue
res[cur_idx[i]] = -1
for i, p in enumerate(processes):
if p is not None and p.is_alive():
continue
if status[i] == ProcStatus.DONE:
try:
p.close()
except Exception:
pass
processes[i] = None
continue
try:
p.close()
except Exception:
pass
is_new_proc = cur_idx[i] == -1
idx = task_ranges[i][0] if is_new_proc else cur_idx[i]
end_idx = task_ranges[i][1]
restarts[i] += 1
processes[i] = Process(
target=eval_f,
args=(
task_mem.name,
res_mem.name,
status_mem.name,
start_time_mem.name,
cur_idx_mem.name,
i,
# skipping the one that takes too long
idx + (0 if is_new_proc else 1),
end_idx,
goal,
task_cnt,
q_type))
processes[i].start()
except Exception:
import traceback
traceback.print_exc()
cur_t = time.time()
time.sleep(timeout)
print_status(res, status, cur_idx, restarts, proc_cnt)
for p in processes:
if p is None:
continue
p.join()
p.close()
for pf in tq[np.argwhere(res == 1)].flatten():
pf = pf.decode('ascii')
print(eval_psfx(pf), '==', postfix_to_infix(pf), pf)
smm.shutdown()
face_cascade = cv2.CascadeClassifier('haarcascade_frontalface_default.xml')
faces = face_cascade.detectMultiScale(gray_img, 1.1, 4)
for (x, y, w, h) in faces:
cv2.rectangle(img, (x, y), (x + w, y + h), (0, 255, 0), 2)
start = time.perf_counter()
if __name__ == '__main__':
cap = cv2.VideoCapture(0)
cap_array = np.array([cap,])
smm = SharedMemoryManager()
smm.start()
cap_memory =
#p1 = multiprocessing.Process(target=do_something, args=[2])
p2 = multiprocessing.Process(target=key_quit)
#p1.start()
p2.start()
#p1.join()
#p2.terminate()
p2.join()
final_memory = shared_memory.SharedMemory(name="cap_mem")
print(f'final : {final_memory.buf[0]}')
class PathFinder:
def __init__(self, maze: Maze = None, memory_manager: SharedMemoryManager = None):
self._maze = maze
self.display = None
self.memory_manager = memory_manager
self.pos = [0, 0]
self._data = {}
if self.memory_manager is None:
self.memory_manager = SharedMemoryManager()
self.memory_manager.start()
def assign_maze(self, maze: Maze):
"""Assign a new maze AND clear data"""
self._maze = maze
# self._data = {}
def start_pygame_display(self):
self.display = Process(
target=pygame_display,
args=[
self._data["maze"].dtype,
self._data["maze_mem"],
self._data["maze"].shape,
self._data["path"].dtype,
self._data["path_mem"],
self._data["path"].shape,
self._data["display_state_mem"],
],
daemon=True,
)
self.display.start()
def solve(self, *args, **kwargs):
start_time = time.time()
time.sleep(0.000001) # prevent division by zero error
def _setup_solve_data(**kwargs):
self._data["node_depth"] = 0
self._data["total_nodes"] = sum([len(x) for x in self._data["nodes"]])
self._data["node_factorial"] = math.factorial(len(self._data["nodes"]))
self._data["step_count"] = 0
if not self._data.get("path_mem", False):
print("[info] setting up shared memory")
path_len = len(self._maze.tiles.flat)
sys.setrecursionlimit(len(self._maze.tiles.flat) * 2)
self._data["path_mem"] = self.memory_manager.SharedMemory(
self._maze.tiles.nbytes * 2
)
self._data["path"] = np.ndarray(
(path_len, 2), dtype=int, buffer=self._data["path_mem"].buf
)
self._data["maze_mem"] = self.memory_manager.SharedMemory(
self._maze.tiles.nbytes
)
self._data["maze"] = np.ndarray(
self._maze.tiles.shape,
dtype=self._maze.tiles.dtype,
buffer=self._data["maze_mem"].buf,
)
# state = running, step_count, step_per_sec
self._data["display_state_mem"] = self.memory_manager.SharedMemory(
(3 * 64)
)
self._data["display_state"] = np.ndarray(
(3,), dtype="uint64", buffer=self._data["display_state_mem"].buf
)
self._data["display_state"][0] = True
self._data["display_state"][1:] = 0
self._data["maze"][:] = self._maze.tiles[:]
self._data["path"].fill(0)
self._config = {
"progress_style": kwargs.get("progress_style", "bar"),
"interval": kwargs.get("interval", 1),
"interval_type": kwargs.get("interval_type", "time"),
"patience": kwargs.get("patience", 0),
}
if not self.display and self._config["progress_style"] == "pygame":
self.start_pygame_display()
elif not self.display.is_alive():
del self.display
self.start_pygame_display()
time.sleep(2)
def _validate_maze(self) -> bool:
if not isinstance(self._maze, Maze):
print(f"[error] {self._maze} is not a {Maze}")
return False
def _check_unwalked(x, y, path_number) -> bool:
if (x, y) in self._data["path"][path_number]:
return False
else:
return True
def _update_progress(self):
def _show_progress(self):
def _node_count():
sys.stdout.write(
f'\r{self._data["node_depth"]} of {self._data["total_nodes"]} nodes'
)
sys.stdout.flush()
def _bar():
printProgressBar(
len(path),
self._data["total_nodes"],
prefix="Finding path...",
suffix="node depth",
)
def _path():
current_time = time.time()
self.show_path(path)
print(
f"Steps: {self._data['step_count']:,} ",
f"Time: {(time.time()-start_time):.0f} sec",
f"\nSteps/second: {int(self._data['step_count']/(current_time-start_time)):,}",
)
def _pygame():
if not self.display:
self.start_pygame_display()
else:
self._data["display_state"][1] = self._data["step_count"]
self._data["display_state"][2] = int(
self._data["step_count"] / (time.time() - start_time)
)
formatted_time = time.strftime("%H:%M:%S", time.gmtime())
print(f"solving... {formatted_time}", end="\r")
# call function based on progress style
{
"node_count": _node_count,
"bar": _bar,
"path": _path,
"pygame": _pygame,
}.get(self._config["progress_style"])()
if self._config["interval_type"] == "time":
if (
time.time() - self._data.get("progress_timer", start_time)
> self._config["interval"]
):
_show_progress(self)
self._data["progress_timer"] = time.time()
elif self._config["interval_type"] == "step_count":
if self._data["step_count"] % self._config["interval"] == 0:
_show_progress(self)
def _traverse(self, coords: tuple, path: set) -> bool:
"""Recursively explores the maze, returns True if the end is found,
returns False when the end cannot be found"""
# self._data["node_depth"] += 1
self._data["step_count"] += 1
path.add(coords)
_update_progress(self)
if (
self._config["patience"]
and time.time() - start_time > self._config["patience"]
):
raise TimeoutError
if coords == self._maze.end:
return True
connections = [c for c in self._data["nodes"][coords] if not c in path]
# recursively traverse each connection
for new_coords in connections:
# set next point in path
self._data["node_depth"] += 1
self._data["path"][self._data["node_depth"]] = new_coords
if _traverse(self, new_coords, path):
return True
else:
# remove point from path
self._data["path"][self._data["node_depth"]] = (0, 0)
self._data["node_depth"] -= 1
path.remove(new_coords)
# Failed to find path at this point
return False
# start pathing the maze
_setup_solve_data(**kwargs)
path = set()
self._data["path"][0] = self._maze.start # first point in path
try:
if _traverse(self, self._maze.start, path):
print("[success] Path found!")
else:
print("[fail] No valid path could be found.")
except TimeoutError:
print("[timeout] took too long to solve")
end_time = time.time()
self._data["display_state"][1] = self._data["step_count"]
self._data["display_state"][2] = int(
self._data["step_count"] / (time.time() - start_time)
)
self._data["solve_time"] = end_time - start_time
def create_nodes(self):
""" Map the maze into a list of nodes"""
self._data["nodes"] = {}
i = 0
map_size = self._maze.x * self._maze.y
for x in range(self._maze.x):
for y in range(self._maze.y):
i += 1
if self._is_node((x, y)):
self._data["nodes"][(x, y)] = []
printProgressBar(
i, map_size, "Creating nodes... ", f"{i}/{map_size}", length=50
)
numer_of_nodes = len(self._data["nodes"])
for i, node in enumerate(self._data["nodes"].keys()):
self._connect_nodes(node)
printProgressBar(
i,
numer_of_nodes - 1,
"Connecting nodes...",
f"{i+1}/{numer_of_nodes}",
length=50,
)
def _connect_nodes(self, node: tuple):
"""connect nodes together"""
x, y = node
def _down():
dy = y
while dy > 0 and self._maze.tiles[x, dy] == 0:
dy -= 1
# if self._is_node((x, dy)):
if (x, dy) in self._data["nodes"]:
self._data["nodes"][(x, y)].append((x, dy))
break
def _left():
dx = x
while dx > 0 and self._maze.tiles[dx, y] == 0:
dx -= 1
# if self._is_node((dx, y)):
if (dx, y) in self._data["nodes"]:
self._data["nodes"][(x, y)].append((dx, y))
break
def _right():
dx = x
while dx < self._maze.x - 1 and self._maze.tiles[dx, y] == 0:
dx += 1
# if self._is_node((dx, y)):
if (dx, y) in self._data["nodes"]:
self._data["nodes"][(x, y)].append((dx, y))
break
def _up():
dy = y
while dy < self._maze.y - 1 and self._maze.tiles[x, dy] == 0:
dy += 1
# if self._is_node((x, dy)):
if (x, dy) in self._data["nodes"]:
self._data["nodes"][(x, y)].append((x, dy))
break
# order determines order of pathing
_down()
_left()
_right()
_up()
def _is_node(self, coords: tuple) -> bool:
# The start and end of a maze are nodes automatically
if coords == self._maze.start or coords == self._maze.end:
return True
x, y = coords
# If all 8 surrounding tiles are clear then ignore
if not self._maze.tiles[x - 1 : x + 2, y - 1 : y + 2].any() > 0:
return False
# Check if the cardinal directions are open
left, right, up, down = False, False, False, False
if x > 0 and self._maze.tiles[x - 1, y] == 0:
left = True
if x < self._maze.x - 1 and self._maze.tiles[x + 1, y] == 0:
right = True
if y > 0 and self._maze.tiles[x, y - 1] == 0:
down = True
if y < self._maze.y - 1 and self._maze.tiles[x, y + 1] == 0:
up = True
# straight paths and deadends are not nodes, but corners are!
for xbool in [left, right]:
for ybool in [up, down]:
if xbool and ybool:
return True
return False
def display_nodes(self):
if not "nodes" in self._data:
print("[error] cannont display nodes, data missing")
return
for x in range(self._maze.x):
line = [" "]
for y in range(self._maze.y):
if self._maze.tiles[x, y] == 1:
line.append("#")
elif (x, y) in self._data["nodes"].keys():
line.append("+")
elif self._maze.tiles[x, y] == 0:
line.append(" ")
else:
line.append("e")
print(" ".join(line))
def show_path(self, path: tuple = None):
"""Prints to terminal the map with the path :n: drawn on it"""
if not path is list:
path = [tuple(p) for p in self._data["path"][:] if tuple(p) != (0, 0)]
# tile type to character lookup dict
characters = {0: " ", 1: "#", 2: "2", 3: "."}
output = []
def _draw_path_between_points(a, b, maze):
# if a is None then b should be the start
if a is None:
maze[b] = 3
else:
dx = 0
if b[0] > a[0]:
dx = 1
elif b[0] < a[0]:
dx = -1
dy = 0
if b[1] > a[1]:
dy = 1
elif b[1] < a[1]:
dy = -1
# paint path onto maze
x, y = a
while (x, y) != b:
x += dx
y += dy
maze[x, y] = 3
if path:
maze = copy.deepcopy(self._maze.tiles)
point_a = None
for point_b in path:
_draw_path_between_points(point_a, point_b, maze)
point_a = point_b
for x in range(maze.shape[0]):
line = [" "]
for y in range(maze.shape[1]):
if (x, y) == path[-1]:
line.append("X")
else:
line.append(characters[maze[x, y]])
output.append(" ".join(line))
if output:
print("\n".join(output))
else:
print("[error] No path to display")
class ParallelSimulation(Simulation):
""" Parallel simulation of Barnes-Hut algorithm realised using shared memory. """
def __init__(self, positions: np.ndarray, velocities: np.ndarray, masses: np.ndarray, params: Namespace):
super().__init__(positions, velocities, masses, params)
self._theta = params.theta
self._init_memory(positions, velocities, masses)
self._init_workers()
atexit.register(self._cleanup)
def _init_memory(self, positions: np.ndarray, velocities: np.ndarray, masses: np.ndarray):
""" Prepares shared memory arrays. """
# setup process that sets up shared memory
self._memory_manager = SharedMemoryManager()
self._memory_manager.start()
max_nodes = self.bodies + 64
# create shared memory buffers
self._positions_shm = self._memory_manager.SharedMemory(positions.nbytes)
self._velocities_shm = self._memory_manager.SharedMemory(velocities.nbytes)
self._accelerations_shm = self._memory_manager.SharedMemory(velocities.nbytes)
self._masses_shm = self._memory_manager.SharedMemory(masses.nbytes)
self._nodes_positions_shm = self._memory_manager.SharedMemory(np.empty((max_nodes, 3), np.float).nbytes)
self._nodes_masses_shm = self._memory_manager.SharedMemory(np.empty((max_nodes, ), np.float).nbytes)
self._nodes_sizes_shm = self._memory_manager.SharedMemory(np.empty((max_nodes, ), np.float).nbytes)
self._nodes_children_types_shm = self._memory_manager.SharedMemory(np.empty((max_nodes, 8), np.int).nbytes)
self._nodes_children_ids_shm = self._memory_manager.SharedMemory(np.empty((max_nodes, 8), np.int).nbytes)
# setup NumPy arrays
self._data = SharedData(
time_step=self.time_step,
theta=self._theta,
gravitational_constant=self.gravitational_constant,
softening=self.softening,
nodes_count=Value('i', 0),
positions=np.ndarray((self.bodies, 3), dtype=np.float, buffer=self._positions_shm.buf),
velocities=np.ndarray((self.bodies, 3), dtype=np.float, buffer=self._velocities_shm.buf),
accelerations=np.ndarray((self.bodies, 3), dtype=np.float, buffer=self._accelerations_shm.buf),
masses=np.ndarray((self.bodies, ), dtype=np.float, buffer=self._masses_shm.buf),
nodes_positions=np.ndarray((max_nodes, 3), dtype=np.float, buffer=self._nodes_positions_shm.buf),
nodes_masses=np.ndarray((max_nodes, ), dtype=np.float, buffer=self._nodes_masses_shm.buf),
nodes_sizes=np.ndarray((max_nodes, ), dtype=np.float, buffer=self._nodes_sizes_shm.buf),
nodes_children_types=np.ndarray((max_nodes, 8), dtype=np.int, buffer=self._nodes_children_types_shm.buf),
nodes_children_ids=np.ndarray((max_nodes, 8), dtype=np.int, buffer=self._nodes_children_ids_shm.buf)
)
# copy data into shared arrays
self._data.positions[:] = positions[:]
self._data.velocities[:] = velocities[:]
self._data.masses[:] = masses[:]
def _init_workers(self):
""" Prepares pool of workers. """
self._pool = Pool(
processes=self._params.processes,
initializer=worker.initialize,
initargs=(self._data, )
)
def _cleanup(self):
""" Cleans up shared memory and pool of workers. """
self._pool.terminate()
self._memory_manager.shutdown()
print('Memory manager was shut down.')
def simulate(self) -> Iterable[Tuple[np.ndarray, np.ndarray, np.ndarray]]:
""" Runs parallel implementation of Barnes-Hut simulation. """
while True:
self._build_octree()
self._update_accelerations()
self._update_positions()
yield self._data.positions, self._data.velocities, self._data.accelerations
def _build_octree(self):
""" Builds octree used in Barnes-Hut. """
global_coords_min = np.repeat(np.min(self._data.positions), 3)
global_coords_max = np.repeat(np.max(self._data.positions), 3)
global_coords_mid = (global_coords_min + global_coords_max) / 2
# manually build first node
self._data.nodes_count.value = 1
self._data.nodes_positions[0] = np.average(self._data.positions, axis=0, weights=self._data.masses)
self._data.nodes_masses[0] = np.sum(self._data.masses)
self._data.nodes_sizes[0] = global_coords_max[0] - global_coords_min[0]
# calculate base octant for each body
bodies_base_octant = np.sum((self._data.positions > global_coords_mid) * [1, 2, 4], axis=1)
tasks_targets = []
tasks_args = []
# build second layer of nodes and collect tasks
for octant in range(8):
coords_min, coords_max = octant_coords(global_coords_min, global_coords_max, octant)
coords_mid = (coords_min + coords_max) / 2
# get indices of bodies in this octant
octant_bodies = np.argwhere(bodies_base_octant == octant).flatten()
# if node is empty or has one body handle it separately
if octant_bodies.size == 0:
self._data.nodes_children_types[0, octant] = OCTANT_EMPTY
continue
if octant_bodies.size == 1:
self._data.nodes_children_types[0, octant] = OCTANT_BODY
self._data.nodes_children_ids[0, octant] = octant_bodies[0]
continue
# create node
node_id = self._data.nodes_count.value
self._data.nodes_count.value = node_id + 1
self._data.nodes_children_types[0, octant] = OCTANT_NODE
self._data.nodes_children_ids[0, octant] = node_id
self._data.nodes_positions[node_id] = np.average(self._data.positions[octant_bodies], axis=0, weights=self._data.masses[octant_bodies])
self._data.nodes_masses[node_id] = np.sum(self._data.masses[octant_bodies])
self._data.nodes_sizes[node_id] = coords_max[0] - coords_min[0]
# split bodies into sub octants
bodies_sub_octant = np.sum((self._data.positions[octant_bodies] > coords_mid) * [1, 2, 4], axis=1)
# create tasks
for i in range(8):
tasks_targets.append((node_id, i))
tasks_args.append((
octant_bodies[bodies_sub_octant == i],
*octant_coords(coords_min, coords_max, i)
))
# run tasks
results = self._pool.starmap(worker.build_octree_branch, tasks_args)
# update references in nodes
for (node_id, i), (sub_node_type, sub_node_id) in zip(tasks_targets, results):
self._data.nodes_children_types[node_id, i] = sub_node_type
self._data.nodes_children_ids[node_id, i] = sub_node_id
def _update_accelerations(self):
""" Calculates accelerations of the bodies. """
if self.bodies < 2:
return
self._pool.map(worker.update_acceleration, range(self.bodies))
def _update_positions(self):
""" Calculates positions of the bodies. """
self._pool.map(worker.update_position, range(self.bodies))
@property
def positions(self) -> np.ndarray:
return self._data.positions
@property
def velocities(self) -> np.ndarray:
return self._data.velocities
@property
def masses(self) -> np.ndarray:
return self._data.masses
@property
def accelerations(self) -> np.ndarray:
return self._data.accelerations
class MonitorApi:
"""The API that monitor clients can access.
The MonitorApi creates and exposes a few "shared resources" that
monitor clients can access. Client access the shared resources through
their SharedMemoryManager which connects to napari.
Exactly what resources we should expose is TBD. Here we are
experimenting with having queue for sending message in each direction,
and a shared dict for sharing data in both directions.
The advantage of a Queue is presumably the other party will definitely
get the message. While the advantage of dict is kind of the opposite,
the other party can check the dict if they want, or they can ignore it.
Again we're not sure what's best yet. But this illustrates some options.
Shared Resources
----------------
napari_data : dict
Napari shares data in this dict for clients to read.
napari_messages : Queue
Napari puts messages in here for clients to read.
napari_shutdown : Event
Napari signals this event when shutting down. Although today napari
does not wait on anything, so typically the client just gets a
connection error when napari goes away, rather than seeing this event.
client_data : Queue
Client shares data in here for napari to read.
client_messages : Queue
Client puts messages in here for napari to read, such as commands.
Notes
-----
The SharedMemoryManager provides the same proxy objects as SyncManager
including list, dict, Barrier, BoundedSemaphore, Condition, Event,
Lock, Namespace, Queue, RLock, Semaphore, Array, Value.
SharedMemoryManager is derived from BaseManager, but it has similar
functionality to SyncManager. See the official Python docs for
multiprocessing.managers.SyncManager.
Numpy can natively use shared memory buffers, something we want to try.
"""
# BaseManager.register() is a bit weird. Not sure now to best deal with
# it. Most ways I tried led to pickling errors, because this class is being run
# in the shared memory server process? Feel free to find a better approach.
_napari_data_dict = dict()
_napari_messages_queue = Queue()
_napari_shutdown_event = Event()
_client_data_dict = dict()
_client_messages_queue = Queue()
@staticmethod
def _napari_data() -> Queue:
return MonitorApi._napari_data_dict
@staticmethod
def _napari_messages() -> Queue:
return MonitorApi._napari_messages_queue
@staticmethod
def _napari_shutdown() -> Event:
return MonitorApi._napari_shutdown_event
@staticmethod
def _client_data() -> Queue:
return MonitorApi._client_data_dict
@staticmethod
def _client_messages() -> Queue:
return MonitorApi._client_messages_queue
def __init__(self):
# RemoteCommands listens to our run_command event. It executes
# commands from the clients.
self.events = EmitterGroup(
source=self, auto_connect=True, run_command=None
)
# We must register all callbacks before we create our instance of
# SharedMemoryManager. The client must do the same thing, but it
# only needs to know the names. We allocate the shared memory.
SharedMemoryManager.register('napari_data', callable=self._napari_data)
SharedMemoryManager.register(
'napari_messages', callable=self._napari_messages
)
SharedMemoryManager.register(
'napari_shutdown', callable=self._napari_shutdown
)
SharedMemoryManager.register('client_data', callable=self._client_data)
SharedMemoryManager.register(
'client_messages', callable=self._client_messages
)
# Start our shared memory server.
self._manager = SharedMemoryManager(
address=('127.0.0.1', SERVER_PORT), authkey=str.encode(AUTH_KEY)
)
self._manager.start()
# Get the shared resources the server created. Clients will access
# these same resources.
self._remote = NapariRemoteAPI(
self._manager.napari_data(),
self._manager.napari_messages(),
self._manager.napari_shutdown(),
self._manager.client_data(),
self._manager.client_messages(),
)
@property
def manager(self) -> SharedMemoryManager:
"""Our shared memory manager.
The wrapper Monitor class accesses this and passes it to the
MonitorService.
Returns
-------
SharedMemoryManager
The manager we created and are using.
"""
return self._manager
def stop(self) -> None:
"""Notify clients we are shutting down.
If we wanted a graceful shutdown, we could wait on "connected"
clients to exit. With a short timeout in case they are hung.
Today we just signal this event and immediately exit. So most of
the time clients just get a connection error. They never see that
this event was set.
"""
self._remote.napari_shutdown.set()
def poll(self):
"""Poll client_messages for new messages."""
assert self._manager is not None
self._process_client_messages()
def _process_client_messages(self) -> None:
"""Process every new message in the queue."""
client_messages = self._remote.client_messages
while True:
try:
message = client_messages.get_nowait()
if not isinstance(message, dict):
LOGGER.warning(
"Ignore message that was not a dict: %s", message
)
continue
# Assume every message is a command that napari should
# execute. We might have other types of messages later.
self.events.run_command(command=message)
except Empty:
return # No commands to process.
def add_napari_data(self, data: dict) -> None:
"""Add data for shared memory clients to read.
Parameters
----------
data : dict
Add this data, replacing anything with the same key.
"""
self._remote.napari_data.update(data)
def send_napari_message(self, message: dict) -> None:
"""Send a message to shared memory clients.
Parameters
----------
message : dict
Message to send to clients.
"""
self._remote.napari_messages.put(message)
class DataManager:
def __init__(self, obs=1000, config=None):
self._datasets = dict()
self.smm = SharedMemoryManager()
self.smm.start()
self.result = Manager().list()
self.conns = dict()
self.obs = obs
self.config = config
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.shutdown()
def shutdown(self):
self.smm.shutdown()
for conn in self.conns.values():
conn.close()
def _add_to_shared_memory(self, nparray: recarray) -> SharedMemory:
"""Internal function to copy an array into shared memory.
Parameters
----------
nparray : recarray
The array to be copied into shared memory.
Returns
-------
SharedMemoryName
The shared memory object.
"""
shm = self.smm.SharedMemory(nparray.nbytes)
array = recarray(shape=nparray.shape, dtype=nparray.dtype, buf=shm.buf)
copyto(array, nparray)
return shm
def _download_dataset(self, dataset: Dataset) -> recarray:
"""Internal function to download the dataset.
Parameters
----------
dataset : Dataset
Dataset information including source, library, table, vars, etc.
Returns
-------
recarray
`numpy.recarry` of the downloaded dataset.
"""
# TODO: generic login data for different data sources
if dataset.source == 'wrds':
usr = self.config.get('wrds_username')
pwd = self.config.get('wrds_password')
# If there exists a connection for the data source, use it!
if (conn := self.conns.get(dataset.source, None)) is None:
module = import_module(f'frds.data.{dataset.source}')
conn = module.Connection(usr=usr, pwd=pwd)
self.conns.update({dataset.source: conn})
df = conn.get_table(library=dataset.library,
table=dataset.table,
columns=dataset.vars,
date_cols=dataset.date_vars,
obs=self.obs)
assert isinstance(df, DataFrame)
return df.to_records(index=False)
class ParallelTransformStep(ProcessingStep[TPayload]):
def __init__(
self,
function: Callable[[Message[TPayload]], TTransformed],
next_step: ProcessingStep[TTransformed],
processes: int,
max_batch_size: int,
max_batch_time: float,
input_block_size: int,
output_block_size: int,
metrics: MetricsBackend,
) -> None:
self.__transform_function = function
self.__next_step = next_step
self.__max_batch_size = max_batch_size
self.__max_batch_time = max_batch_time
self.__shared_memory_manager = SharedMemoryManager()
self.__shared_memory_manager.start()
self.__pool = Pool(
processes,
initializer=parallel_transform_worker_initializer,
context=multiprocessing.get_context("spawn"),
)
self.__input_blocks = [
self.__shared_memory_manager.SharedMemory(input_block_size)
for _ in range(processes)
]
self.__output_blocks = [
self.__shared_memory_manager.SharedMemory(output_block_size)
for _ in range(processes)
]
self.__batch_builder: Optional[BatchBuilder[TPayload]] = None
self.__results: Deque[Tuple[MessageBatch[TPayload], AsyncResult[Tuple[
int, MessageBatch[TTransformed]]], ]] = deque()
self.__metrics = metrics
self.__batches_in_progress = Gauge(metrics, "batches_in_progress")
self.__pool_waiting_time: Optional[float] = None
self.__closed = False
def handle_sigchld(signum: int, frame: Any) -> None:
# Terminates the consumer if any child process of the
# consumer is terminated.
# This is meant to detect the unexpected termination of
# multiprocessor pool workers.
if not self.__closed:
self.__metrics.increment("sigchld.detected")
raise ChildProcessTerminated()
signal.signal(signal.SIGCHLD, handle_sigchld)
def __submit_batch(self) -> None:
assert self.__batch_builder is not None
batch = self.__batch_builder.build()
logger.debug("Submitting %r to %r...", batch, self.__pool)
self.__results.append((
batch,
self.__pool.apply_async(
parallel_transform_worker_apply,
(self.__transform_function, batch, self.__output_blocks.pop()),
),
))
self.__batches_in_progress.increment()
self.__metrics.timing("batch.size.msg", len(batch))
self.__metrics.timing("batch.size.bytes", batch.get_content_size())
self.__batch_builder = None
def __check_for_results(self, timeout: Optional[float] = None) -> None:
input_batch, result = self.__results[0]
# If this call is being made in a context where it is intended to be
# nonblocking, checking if the result is ready (rather than trying to
# retrieve the result itself) avoids costly synchronization.
if timeout == 0 and not result.ready():
# ``multiprocessing.TimeoutError`` (rather than builtin
# ``TimeoutError``) maintains consistency with ``AsyncResult.get``.
raise multiprocessing.TimeoutError()
i, output_batch = result.get(timeout=timeout)
# TODO: This does not handle rejections from the next step!
for message in output_batch:
self.__next_step.poll()
self.__next_step.submit(message)
if i != len(input_batch):
logger.warning(
"Received incomplete batch (%0.2f%% complete), resubmitting...",
i / len(input_batch) * 100,
)
# TODO: This reserializes all the ``SerializedMessage`` data prior
# to the processed index even though the values at those indices
# will never be unpacked. It probably makes sense to remove that
# data from the batch to avoid unnecessary serialization overhead.
self.__results[0] = (
input_batch,
self.__pool.apply_async(
parallel_transform_worker_apply,
(
self.__transform_function,
input_batch,
output_batch.block,
i,
),
),
)
return
logger.debug("Completed %r, reclaiming blocks...", input_batch)
self.__input_blocks.append(input_batch.block)
self.__output_blocks.append(output_batch.block)
self.__batches_in_progress.decrement()
del self.__results[0]
def poll(self) -> None:
self.__next_step.poll()
while self.__results:
try:
self.__check_for_results(timeout=0)
except multiprocessing.TimeoutError:
if self.__pool_waiting_time is None:
self.__pool_waiting_time = time.time()
else:
current_time = time.time()
if current_time - self.__pool_waiting_time > LOG_THRESHOLD_TIME:
logger.warning(
"Waited on the process pool longer than %d seconds. Waiting for %d results. Pool: %r",
LOG_THRESHOLD_TIME,
len(self.__results),
self.__pool,
)
self.__pool_waiting_time = current_time
break
else:
self.__pool_waiting_time = None
if self.__batch_builder is not None and self.__batch_builder.ready():
self.__submit_batch()
def __reset_batch_builder(self) -> None:
try:
input_block = self.__input_blocks.pop()
except IndexError as e:
raise MessageRejected("no available input blocks") from e
self.__batch_builder = BatchBuilder(
MessageBatch(input_block),
self.__max_batch_size,
self.__max_batch_time,
)
def submit(self, message: Message[TPayload]) -> None:
assert not self.__closed
if self.__batch_builder is None:
self.__reset_batch_builder()
assert self.__batch_builder is not None
try:
self.__batch_builder.append(message)
except ValueTooLarge as e:
logger.debug("Caught %r, closing batch and retrying...", e)
self.__submit_batch()
# This may raise ``MessageRejected`` (if all of the shared memory
# is in use) and create backpressure.
self.__reset_batch_builder()
assert self.__batch_builder is not None
# If this raises ``ValueTooLarge``, that means that the input block
# size is too small (smaller than the Kafka payload limit without
# compression.)
self.__batch_builder.append(message)
def close(self) -> None:
self.__closed = True
if self.__batch_builder is not None and len(self.__batch_builder) > 0:
self.__submit_batch()
def terminate(self) -> None:
self.__closed = True
logger.debug("Terminating %r...", self.__pool)
self.__pool.terminate()
logger.debug("Shutting down %r...", self.__shared_memory_manager)
self.__shared_memory_manager.shutdown()
logger.debug("Terminating %r...", self.__next_step)
self.__next_step.terminate()
def join(self, timeout: Optional[float] = None) -> None:
deadline = time.time() + timeout if timeout is not None else None
logger.debug("Waiting for %s batches...", len(self.__results))
while self.__results:
self.__check_for_results(
timeout=max(deadline -
time.time(), 0) if deadline is not None else None)
self.__pool.close()
logger.debug("Waiting for %s...", self.__pool)
# ``Pool.join`` doesn't accept a timeout (?!) but this really shouldn't
# block for any significant amount of time unless something really went
# wrong (i.e. we lost track of a task)
self.__pool.join()
self.__shared_memory_manager.shutdown()
self.__next_step.close()
self.__next_step.join(
timeout=max(deadline -
time.time(), 0) if deadline is not None else None)
def _batched_train_test(self):
#batch_size = len(self.GPU_LIST)*self.model_per_gpu
smm = SharedMemoryManager()
smm.start()
X_shared = _to_shared_mem(smm, self.X)
w_shared = _to_shared_mem(smm, self.w)
delta_shared = _to_shared_mem(smm, self.delta)
with Manager() as manager:
param_dict_mngd = manager.dict({
'param_dicts_train':
self.param_dicts_train,
'param_dicts_test':
self.param_dicts_test,
'data_idx':
self.data_idx,
'X_shared_name':
X_shared.name,
'X_shape':
self.X.shape,
'X_dtype':
self.X.dtype,
'w_shared_name':
w_shared.name,
'w_shape':
self.w.shape,
'w_dtype':
self.w.dtype,
'delta_shared_name':
delta_shared.name,
'delta_shape':
self.delta.shape,
'delta_dtype':
self.delta.dtype,
'batch_size':
self.batch_size,
'batch_idx':
None,
'test_block_size':
self.test_block_size
})
rslt_mngd = manager.list([0] * len(self.param_dicts_test))
for batch_idx in tqdm(range(
int(len(self.param_dicts_train) / self.batch_size) + 1),
desc="batched cross validation"):
param_dict_mngd['batch_idx'] = batch_idx
'''
_run_batch_process(param_dict_mngd, rslt_mngd)
print (rslt_mngd)
raise
'''
p = mp.Process(target=_run_batch_process,
args=(param_dict_mngd, rslt_mngd))
p.start()
p.join()
p.terminate()
smm.shutdown()
self.rslts = list(rslt_mngd)
class DetectionMonitor(object):
"""
A global manager that creates camera workdirs based on the camera's configuration file,
initializes many services, and defines basic state control functions.
"""
def __init__(self,
cfgs: List[VideoConfig],
scfg: ServerConfig,
stream_path: Path,
sample_path: Path,
frame_path: Path,
region_path: Path,
offline_path: Path = None,
build_pool=True) -> None:
super().__init__()
# self.cfgs = I.load_video_config(cfgs)[-1:]
# self.cfgs = I.load_video_config(cfgs)
# self.cfgs = [c for c in self.cfgs if c.enable]
# self.cfgs = [c for c in cfgs if enable_options[c.index]]
self.scfg = scfg
self.cfgs = cfgs
self.quit = False
# Communication Pipe between detector and stream receiver
self.pipes = [Manager().Queue(c.max_streams_cache) for c in self.cfgs]
self.time_stamp = generate_time_stamp('%m%d')
self.stream_path = stream_path / self.time_stamp
self.sample_path = sample_path / self.time_stamp
self.frame_path = frame_path / self.time_stamp
self.region_path = region_path / self.time_stamp
self.offline_path = offline_path
self.process_pool = None
self.thread_pool = None
self.shut_down_event = Manager().Event()
self.shut_down_event.clear()
self.scheduler = BackgroundScheduler()
if build_pool:
# build service
# pool_size = min(len(cfgs) * 2, cpu_count() - 1)
# self.process_pool = Pool(processes=pool_size)
self.process_pool = Pool(processes=len(self.cfgs) * 5)
self.thread_pool = ThreadPoolExecutor()
# self.clean()
self.stream_receivers = [
stream.StreamReceiver(self.stream_path / str(c.index),
offline_path, c, self.pipes[idx])
for idx, c in enumerate(self.cfgs)
]
self.scheduler.add_job(self.notify_shut_down,
'cron',
month=self.scfg.cron['end']['month'],
day=self.scfg.cron['end']['day'],
hour=self.scfg.cron['end']['hour'],
minute=self.scfg.cron['end']['minute'])
self.scheduler.start()
self.frame_cache_manager = SharedMemoryManager()
self.frame_cache_manager.start()
def monitor(self):
self.call()
self.wait()
# def set_runtime(self, runtime):
# self.runtime = runtime
def shut_down_from_keyboard(self):
logger.info('Click Double Enter to shut down system.')
while True and not self.shut_down_event.is_set():
c = sys.stdin.read(1)
logger.info(c)
if c == '\n':
self.notify_shut_down()
break
# if keycode == Key.enter:
# self.shut_down_event.set()
def shut_down_after(self, runtime=-1):
if runtime == -1:
return
logger.info('System will exit after [{}] seconds'.format(runtime))
threading.Timer(runtime, self.notify_shut_down).start()
def notify_shut_down(self):
if not self.shut_down_event.is_set():
self.shut_down_event.set()
def listen(self):
# Listener(on_press=self.shut_down_from_keyboard).start()
threading.Thread(target=self.shut_down_from_keyboard,
daemon=True).start()
logger.info(
'*******************************Monitor: Listening exit event********************************'
)
# if self.runtime != -1:
# time.sleep(self.runtime)
# else:
# input('')
self.shut_down_event.wait()
logger.info(
'*******************************Monitor: preparing exit system********************************'
)
self.cancel()
def cancel(self):
pass
def call(self):
for i, cfg in enumerate(self.cfgs):
# clean all legacy streams and candidates files before initialization
self.init_stream_receiver(i)
self.init_detection(cfg, i)
def wait(self):
logger.info('Wait processes done.')
if self.process_pool is not None:
self.process_pool.close()
self.process_pool.join()
logger.info('Closed Pool')
def init_detection(self, cfg, i):
if self.process_pool is not None:
self.process_pool.apply_async(detect, (
self.stream_path / str(cfg.index),
self.region_path / str(cfg.index),
self.pipes[i],
cfg,
))
def init_stream_receiver(self, i):
if self.process_pool is not None:
return self.process_pool.apply_async(
self.stream_receivers[i].receive_online)
def clean(self):
clean_dir(self.sample_path)
clean_dir(self.stream_path)
clean_dir(self.region_path)
class MonitorApi:
"""The API that monitor clients can access.
MonitorApi will execute commands from the clients when it is polled.
Shared Resources
----------------
Clients can access these shared resources via their SharedMemoryManager
that connects to napari.
shutdown : Event
Signaled when napari is shutting down.
commands : Queue
Client can put "commands" on this queue.
client_messages : Queue
Clients receive messages on this queue.
data : dict
Generic data from monitor.add()
Notes
-----
The SharedMemoryManager provides the same proxy objects as SyncManager
including list, dict, Barrier, BoundedSemaphore, Condition, Event,
Lock, Namespace, Queue, RLock, Semaphore, Array, Value.
SharedMemoryManager is derived from BaseManager, but it has similar
functionality to SyncManager. See the docs for
multiprocessing.managers.SyncManager.
Parameters
----------
Layer : LayerList
The viewer's layers.
"""
# BaseManager.register() is a bit weird. There must be a better wa
# to do this. But most things including lambda result in pickling
# errors due to multiprocessing stuff.
#
# So we create these and then use staticmethods as our callables.
_napari_shutting_down_event = Event()
_commands_queue = Queue()
_client_messages_queue = Queue()
_data_dict = dict()
@staticmethod
def _napari_shutting_down() -> Event:
return MonitorApi._napari_shutting_down_event
@staticmethod
def _commands() -> Queue:
return MonitorApi._commands_queue
@staticmethod
def _client_messages() -> Queue:
return MonitorApi._client_messages_queue
@staticmethod
def _data() -> dict:
return MonitorApi._data_dict
def __init__(self):
# We expose the run_command event so RemoteCommands can hook to it,
# so it can execute commands we receive from clients.
self.events = EmitterGroup(source=self,
auto_connect=True,
run_command=None)
# Must register all callbacks before we create our instance of
# SharedMemoryManager.
SharedMemoryManager.register('napari_shutting_down',
callable=self._napari_shutting_down)
SharedMemoryManager.register('commands', callable=self._commands)
SharedMemoryManager.register('client_messages',
callable=self._client_messages)
SharedMemoryManager.register('data', callable=self._data)
# We ask for port 0 which means let the OS choose a port. We send
# the chosen port to the client in its NAPARI_MON_CLIENT variable.
self._manager = SharedMemoryManager(address=('127.0.0.1', 0),
authkey=str.encode('napari'))
self._manager.start()
# Get the shared resources.
self._remote = NapariRemoteAPI(
self._manager.napari_shutting_down(),
self._manager.commands(),
self._manager.client_messages(),
self._manager.data(),
)
@property
def manager(self) -> SharedMemoryManager:
"""Our shared memory manager."""
return self._manager
def stop(self) -> None:
"""Notify clients we are shutting down."""
self._remote.napari_shutting_down.set()
def poll(self):
"""Poll the MonitorApi for new commands, etc."""
assert self._manager is not None
self._process_commands()
def _process_commands(self) -> None:
"""Process every new command in the queue."""
while True:
try:
command = self._remote.commands.get_nowait()
if not isinstance(command, dict):
LOGGER.warning("Command was not a dict: %s", command)
continue
self.events.run_command(command=command)
except Empty:
return # No commands to process.
def add(self, data: dict) -> None:
"""Add data for shared memory clients to read.
Parameters
----------
data : dict
Add this data, replacing anything with the same key.
"""
self._remote.data.update(data)
def send(self, message: dict) -> None:
"""Send a message to shared memory clients."""
LOGGER.info("MonitorApi.send: %s", json.dumps(message))
self._remote.client_messages.put(message)
class DataManager(Singleton):
"""DataManager loads data from sources and manages the shared memory"""
def __init__(self, obs=-1, config=None):
self._datasets = dict()
self.smm = SharedMemoryManager()
self.smm.start()
self.conns = dict()
self.obs = obs
self.config = config
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.shutdown()
def shutdown(self):
"""Shutdown the shared memory manager and all data connections"""
self.smm.shutdown()
for conn in self.conns.values():
conn.close()
def _add_to_shared_memory(self, nparray: np.recarray) -> SharedMemory:
"""Internal function to copy an array into shared memory.
Parameters
----------
nparray : np.recarray
The array to be copied into shared memory.
Returns
-------
SharedMemory
The shared memory object.
"""
shm = self.smm.SharedMemory(nparray.nbytes)
array = np.recarray(nparray.shape, dtype=nparray.dtype, buf=shm.buf)
np.copyto(array, nparray)
return shm
def _download_dataset(self, dataset: Dataset) -> pd.DataFrame:
"""Internal function to download the dataset.
Parameters
----------
dataset : Dataset
Dataset information including source, library, table, vars, etc.
Returns
-------
pd.DataFrame
DataFrame of the downloaded dataset.
"""
# TODO: generic login data for different data sources
if dataset.source == "wrds":
usr = self.config.get("wrds_username")
pwd = self.config.get("wrds_password")
# If there exists a connection for the data source, use it!
if (conn := self.conns.get(dataset.source, None)) is None:
module = import_module(f"frds.data.{dataset.source}")
conn = module.Connection(usr=usr, pwd=pwd)
self.conns.update({dataset.source: conn})
return conn.get_table(
library=dataset.library,
table=dataset.table,
columns=dataset.vars,
date_cols=dataset.date_vars,
obs=self.obs,
)