def producer(conn):
# os.environ["PYTHONWARNINGS"] = "ignore"
feed_shm_name = '{}_{}_{}'.format('test', os.getpid(),
threading.currentThread().ident)
print('input shm name : {}'.format(feed_shm_name))
feed_shm = SharedMemory(name=feed_shm_name, create=True, size=2 * 4)
feed_shm_arr = np.ndarray((1, 2), dtype=np.float32, buffer=feed_shm.buf)
input_arr = np.random.random((1, 2)).astype(np.float32)
feed_shm_arr[:] = input_arr[:]
conn.send(feed_shm_name)
result_shm_name = conn.recv()
result_shm = SharedMemory(name=result_shm_name)
result_shm_arr = np.ndarray((1, 2),
dtype=np.float32,
buffer=result_shm.buf)
print('Output array : {}'.format(result_shm_arr))
conn.send('exit')
del result_shm_arr
result_shm.close()
conn.recv()
del feed_shm_arr
feed_shm.close()
feed_shm.unlink()
print('clean and exit')
return
def __init__(self, args: RemoteArgs, progress_cb, is_master):
self._args = args
self._progress_cb = progress_cb
self._make_verifier()
shm = SharedMemory(args.shm_name, False)
try:
self._setup_shm(shm.buf)
self._process()
self._done_workers.fetch_add(1)
except:
# set to a negative value to mark failure
self._read_idx.fetch_add(-args.inputs.shape[0]**2)
raise
finally:
self._read_idx = None
self._done_workers = None
self._nr_adv_verifier = None
self._nr_adv_ref = None
self._output = None
shm.close()
# manual cleanup due to a bug (https://bugs.python.org/issue39959 )
if not is_master and sys.version_info <= (3, 8, 2):
from multiprocessing import shared_memory
if shared_memory._USE_POSIX:
from multiprocessing.resource_tracker import unregister
unregister(shm._name, "shared_memory")
def chunk_worker(left_spec, right_spec, pairs_slice):
# recover the SharedMemory bloc
left_shm = SharedMemory(left_spec['name'])
right_shm = SharedMemory(right_spec['name'])
# Create the np.recarray from the buffer of the shared memory
left_arr = np.recarray(shape=left_spec['shape'],
dtype=left_spec['dtype'],
buf=left_shm.buf)
right_arr = np.recarray(shape=right_spec['shape'],
dtype=right_spec['dtype'],
buf=right_shm.buf)
#print (left_arr[0][2], right_arr[0][2])
results = []
for idx_pair in pairs_slice:
geom1 = wkt.loads(left_arr[idx_pair[0]][2])
geom2 = wkt.loads(right_arr[idx_pair[1]][2])
pt1, pt2 = ops.nearest_points(geom1, geom3)
dist, a1, a2 = V_inv((pt1.y, pt1.x), (pt2.y, pt2.x))
dist = dist * 1000 #m
results.append((idx_pair[0], idx_pair[1], dist))
return results
def random_drop(vocab, embedding: Embedding, percent, vocab_size):
weights_mem = SharedMemory(embedding.embedding_memory_name)
W = np.ndarray(shape=embedding.embedding_memory_shape,
dtype=embedding.embedding_memory_dtype,
buffer=weights_mem.buf)
w2i_mem = SharedMemory(embedding.w2i_memory_name)
w2i = embedding.buff_to_dict(w2i_mem, embedding.w2i_memory_size)
dropped_words = {}
for c in vocab:
if c not in dropped_words:
dropped_words[c] = []
existing_words = []
for word in vocab[c]: # iterating over words
try:
_ = W[w2i[word]]
existing_words.append(word)
except KeyError:
continue
size = existing_words.__len__()
rm_num = math.floor(size * percent)
vocab_size -= rm_num
for i in range(rm_num):
rng = random.randint(0, size - i - 1)
dropped_words[c].append(existing_words[rng])
del existing_words[rng]
vocab[c] = existing_words
return dropped_words
def sensor_read_process(raw_shm: shared_memory.SharedMemory,
proc_shm: shared_memory.SharedMemory, r_lock: Lock,
p_lock: Lock):
script_dir = Path(__file__).parent
device_file = script_dir.parent.joinpath("um7_A500CNP8.json")
assert device_file.exists(
), f"Device file with connection info: {device_file} does not exist!"
um7 = UM7Serial(device=str(device_file))
for packet in um7.recv_broadcast(flush_buffer_on_start=False):
packet_bytes = bytes(json.dumps(packet.__dict__), encoding='utf-8')
assert len(
packet_bytes
) <= BUFFER_SIZE, f"Packet cannot be serialized, increase `BUFFER` size at least up to {len(packet_bytes)}"
if isinstance(packet, UM7AllRawPacket):
r_lock.acquire()
raw_shm.buf[:] = b' ' * BUFFER_SIZE
raw_shm.buf[:len(packet_bytes)] = packet_bytes
r_lock.release()
# logging.warning(f"[SR][RAW] -> {packet}")
elif isinstance(packet, UM7AllProcPacket):
p_lock.acquire()
proc_shm.buf[:] = b' ' * BUFFER_SIZE
proc_shm.buf[:len(packet_bytes)] = packet_bytes
p_lock.release()
def attach_shm_array(
token: tuple[str, str, tuple[str, str]],
size: int = _default_size,
readonly: bool = True,
) -> ShmArray:
'''
Attach to an existing shared memory array previously
created by another process using ``open_shared_array``.
No new shared mem is allocated but wrapper types for read/write
access are constructed.
'''
token = _Token.from_msg(token)
key = token.shm_name
if key in _known_tokens:
assert _Token.from_msg(_known_tokens[key]) == token, "WTF"
# attach to array buffer and view as per dtype
shm = SharedMemory(name=key)
shmarr = np.ndarray((size, ), dtype=token.dtype, buffer=shm.buf)
shmarr.setflags(write=int(not readonly))
first = SharedInt(
shm=SharedMemory(
name=token.shm_first_index_name,
create=False,
size=4, # std int
), )
last = SharedInt(
shm=SharedMemory(
name=token.shm_last_index_name,
create=False,
size=4, # std int
), )
# make sure we can read
first.value
sha = ShmArray(
shmarr,
first,
last,
shm,
)
# read test
sha.array
# Stash key -> token knowledge for future queries
# via `maybe_opepn_shm_array()` but only after we know
# we can attach.
if key not in _known_tokens:
_known_tokens[key] = token
# "close" attached shm on process teardown
tractor._actor._lifetime_stack.callback(sha.close)
return sha
def __init__(self,
chunk_size: int=DEFAULT_CHUNK_SIZE,
maxsize: int=DEFAULT_MAXSIZE,
serializer=None,
integrity_check: bool=False,
deadlock_check: bool=False,
deadlock_immanent_check: bool=True,
watermark_check: bool = False,
use_semaphores: bool = True,
verbose: bool=False):
ctx = mp.get_context() # TODO: What is the proper type hint here?
super().__init__(maxsize, ctx=ctx)
self.qid: int = self.__class__.qid_counter
self.__class__.qid_counter += 1
self.verbose: bool = verbose
if self.verbose:
print("Starting ShmQueue qid=%d pid=%d chunk_size=%d maxsize=%d." % (self.qid, os.getpid(), chunk_size, maxsize), file=sys.stderr, flush=True) # ***
self.chunk_size: int = min(chunk_size, self.__class__.MAX_CHUNK_SIZE) \
if chunk_size > 0 else self.__class__.MAX_CHUNK_SIZE
self.maxsize: int = maxsize if maxsize > 0 else self.__class__.DEFAULT_MAXSIZE
self.serializer = serializer or pickle
self.integrity_check: bool = integrity_check
self.deadlock_check: bool = deadlock_check
self.deadlock_immanent_check: bool = deadlock_immanent_check
self.watermark_check: bool = watermark_check
self.chunk_watermark: int = 0
self.mid_counter: int = 0
self.producer_lock = ctx.Lock()
self.free_list_lock = ctx.Lock()
self.msg_list_lock = ctx.Lock()
self.use_semaphores: bool = use_semaphores
if not use_semaphores:
# Put the None case first to make mypy happier.
self.free_list_semaphore: typing.Optional[typing.Any] = None # TODO: what is the type returned by ctx.Semaphore(0)?
self.msg_list_semaphore: typing.Optional[typing.Any] = None
else:
self.free_list_semaphore = ctx.Semaphore(0)
self.msg_list_semaphore = ctx.Semaphore(0)
self.list_heads: SharedMemory = SharedMemory(create=True, size=self.__class__.LIST_HEAD_SIZE * 2)
self.init_list_head(self.__class__.FREE_LIST_HEAD)
self.init_list_head(self.__class__.MSG_LIST_HEAD)
self.block_locks: typing.List[typing.Any] = [ctx.Lock()] * maxsize # TODO: what is the type returned by ctx.Lock()?
self.data_blocks: typing.List[SharedMemory] = []
block_id: int
for block_id in range(maxsize):
self.data_blocks.append(SharedMemory(create=True, size=self.__class__.META_BLOCK_SIZE + self.chunk_size))
self.add_free_block(block_id)
async def teardown(**kwargs):
object_ids = kwargs.get('object_ids')
for object_id in object_ids:
try:
shm = SharedMemory(name=object_id)
shm.unlink()
await asyncio.sleep(0)
except FileNotFoundError:
pass
def open_shm_array(
key: Optional[str] = None,
size: int = _default_size,
dtype: Optional[np.dtype] = None,
readonly: bool = False,
) -> ShmArray:
"""Open a memory shared ``numpy`` using the standard library.
This call unlinks (aka permanently destroys) the buffer on teardown
and thus should be used from the parent-most accessor (process).
"""
# create new shared mem segment for which we
# have write permission
a = np.zeros(size, dtype=dtype)
a['index'] = np.arange(len(a))
shm = SharedMemory(name=key, create=True, size=a.nbytes)
array = np.ndarray(a.shape, dtype=a.dtype, buffer=shm.buf)
array[:] = a[:]
array.setflags(write=int(not readonly))
token = _make_token(key=key, dtype=dtype)
# create single entry arrays for storing an first and last indices
first = SharedInt(shm=SharedMemory(
name=token.shm_first_index_name,
create=True,
size=4, # std int
))
last = SharedInt(shm=SharedMemory(
name=token.shm_last_index_name,
create=True,
size=4, # std int
))
last.value = first.value = int(_secs_in_day)
shmarr = ShmArray(
array,
first,
last,
shm,
)
assert shmarr._token == token
_known_tokens[key] = shmarr.token
# "unlink" created shm on process teardown by
# pushing teardown calls onto actor context stack
tractor._actor._lifetime_stack.callback(shmarr.close)
tractor._actor._lifetime_stack.callback(shmarr.destroy)
return shmarr
def alloc(self, sm_name=None, create=True):
sm_name = sm_name or _make_filename()
try:
sm = SharedMemory(sm_name, create, self.__total_size)
except:
sm = SharedMemory(sm_name, not create, self.__total_size)
buffer = sm.buf
res = Dict()
offset = 0
for name, length, c_type, size in self.__vars:
res[name] = np.ndarray((length, ), c_type, buffer, offset=offset)
offset += size
return sm, res
def category_center(vocab, embedding: Embedding, percent, vocab_size):
weights_mem = SharedMemory(embedding.embedding_memory_name)
W = np.ndarray(shape=embedding.embedding_memory_shape,
dtype=embedding.embedding_memory_dtype,
buffer=weights_mem.buf)
w2i_mem = SharedMemory(embedding.w2i_memory_name)
w2i = embedding.buff_to_dict(w2i_mem, embedding.w2i_memory_size)
dropped_words = {}
for c in vocab: # c is the category
if c not in dropped_words: # populating dropped_words with empty lists
dropped_words[c] = []
mean = None
for word in vocab[c]: # iterating over words
try:
# getting coefficients of the words of c
if mean is None:
mean = np.array([W[w2i[word]]])
else:
vector = [W[w2i[word]]]
mean = np.append(mean, vector, axis=0)
except KeyError:
continue
# getting the category center
mean = np.mean(mean)
ranks = None
# calculating distances from the category centers and sorting by that afterwards
for i, word in enumerate(vocab[c]):
try:
if ranks is None:
ranks = np.array([[i, _distance(mean, W[w2i[word]])]])
else:
ranks = np.append(ranks,
[[i, _distance(mean, W[w2i[word]])]],
axis=0)
except KeyError:
continue
ranks = ranks[ranks[:, 1].argsort(), :]
drop = math.floor(ranks.shape[0] * percent)
vocab_size -= drop
word_list = []
for i in range(drop):
index = int(ranks[-i - 1, 0])
dropped_words[c].append(vocab[c][index])
word_list.append(vocab[c][index])
for w in word_list:
vocab[c].remove(w)
return dropped_words
def __init__(self, name, queue, out_queue, shm, max_retries=5,
logging_queue=None, dl_timeout=10):
super().__init__(name=name)
self.q = queue
self.o_q = out_queue
self.session = requests.session()
self.session.headers.update({
'User-Agent': 'EpicGamesLauncher/10.18.6-14188424+++Portal+Release-Live Windows/10.0.18363.1.256.64bit'
})
self.max_retries = max_retries
self.shm = SharedMemory(name=shm)
self.log_level = logging.getLogger().level
self.logging_queue = logging_queue
self.dl_timeout = float(dl_timeout) if dl_timeout else 10.0
def work_with_shared_memory(shm_name, shape, dtype):
print(f'With SharedMemory: {current_process()=}')
# Locate the shared memory by its name
shm = SharedMemory(shm_name)
# Create the np.recarray from the buffer of the shared memory
np_array = np.recarray(shape=shape, dtype=dtype, buf=shm.buf)
return np.nansum(np_array.val)
def _write_init(self):
# keep last 8 bytes to record actual memory size
self.shm = shm = SharedMemory(name=self._object_id,
create=True,
size=self._size + 8)
self._write_actual_size()
self._buffer = self._mv = shm.buf
def _init_memory_worker(recording, arrays, shm_names, shapes, dtype):
# create a local dict per worker
worker_ctx = {}
if isinstance(recording, dict):
from spikeinterface.core import load_extractor
worker_ctx['recording'] = load_extractor(recording)
else:
worker_ctx['recording'] = recording
worker_ctx['dtype'] = np.dtype(dtype)
if arrays is None:
# create it from share memory name
from multiprocessing.shared_memory import SharedMemory
arrays = []
# keep shm alive
worker_ctx['shms'] = []
for i in range(len(shm_names)):
shm = SharedMemory(shm_names[i])
worker_ctx['shms'].append(shm)
arr = np.ndarray(shape=shapes[i], dtype=dtype, buffer=shm.buf)
arrays.append(arr)
worker_ctx['arrays'] = arrays
return worker_ctx
def load(q_comb, layerNo, layerSN):
# layerNo: the number in the height axis used to be load
all_temp_tif = glob.glob("temp*.tif")
shm_list = list()
shared_img_list = list()
core_no = mp.cpu_count() - 2
for n in range(core_no):
shm_list.append(SharedMemory(create=True, size=2 * layerNo * 5 * 9261))
shared_img_list.append(
np.ndarray(shape=(layerNo, 9261, 5),
dtype="uint16",
buffer=shm_list[n].buf))
for idx, tif_name in enumerate(all_temp_tif):
img = TFF.imread(tif_name,
key=range(layerSN, layerSN + core_no * layerNo))
for n in range(core_no):
shared_img_list[n] = np.ndarray(shape=(layerNo, img.shape[1],
img.shape[2]),
dtype=img.dtype,
buffer=shm_list[n].buf)
np.copyto(shared_img_list[n],
img[n * layerNo:(n + 1) * layerNo, :, :])
for idx, a_pipe in enumerate(q_comb):
a_pipe[0].send([
tif_name, shm_list[idx].name, shared_img_list[idx].shape, idx
])
def is_p(i, j, config: Config, embedding_memory: MemoryInfo, lamb: int):
lambdas = {l + 1: 0 for l in range(lamb)}
# Embedding
weights_mem = SharedMemory(embedding_memory.name)
w = np.ndarray(shape=embedding_memory.shape, buffer=weights_mem.buf)
V_1 = np.array([w[:, j]])
V_2 = np.array([np.arange(V_1.shape[1])])
V = np.append(V_1, V_2, axis=0)
V_sorted = V[:, V[0, :].argsort()]
S = set(config.semantic_categories.categories.vocab[
config.semantic_categories.categories.i2c[i]])
n_i = config.semantic_categories.categories.vocab[
config.semantic_categories.categories.i2c[i]].__len__()
for l in range(lamb):
v_p = V_p(V_sorted, n_i, l + 1, config)
v_n = V_n(V_sorted, n_i, l + 1, config)
IS_p = S.intersection(v_p).__len__() / n_i * 100
IS_n = S.intersection(v_n).__len__() / n_i * 100
lambdas[l + 1] = max(IS_p, IS_n)
return lambdas
def load_file(self, *args, **kwargs):
shm = SharedMemory(create=True, size=5 * np.dtype('<U64').itemsize)
# get array metadata, shared memory buffer name, dtype, and shape
array_metadata = np.ndarray(shape=(5,), dtype=np.dtype('<U64'), buffer=shm.buf)
importer_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '_inscopix_importer.py')
file_path = self.widget.line_edit_path.text()
cmd = ['python', importer_path, '--isx-path', file_path, '--shm-meta-array-name', shm.name]
print(cmd)
proc = Popen(
cmd,
env=os.environ.copy()
)
proc.wait()
# read the metadata
name = array_metadata[0]
dtype = array_metadata[1]
shape = array_metadata[2]
# open the shared buffer
existing_shm = SharedMemory(name=array_metadata[0])
shape = tuple(map(int, shape.split(',')))
_imgseq = np.ndarray(shape, dtype=np.dtype(dtype), buffer=existing_shm.buf)
imgseq = np.zeros(shape=shape, dtype=np.dtype(dtype))
imgseq[:] = _imgseq[:]
d = \
{
'fps': float(array_metadata[3]),
'origin': array_metadata[4],
'date': '00000000_000000'
}
imgdata = ImgData(imgseq, d)
self.vi.viewer.workEnv = ViewerWorkEnv(imgdata)
self.vi.update_workEnv()
existing_shm.close()
def __init__(
self,
points,
iD,
kD,
vectors_UID,
tree_UID,
maxsize,
pipe,
lock,
distance_function=(lambda v: np.sum(v**2)),
vector_function=(lambda p, n: p - n),
square_distances=True,
):
super(Horse, self).__init__()
self.distance_function = distance_function
self.vector_function = vector_function
self.points = points
if square_distances:
self.iD = iD**2
self.kD = kD**2
else:
self.iD = iD
self.kD = kD
self.maxsize = maxsize
self.reached_bool = np.zeros(len(points), dtype=bool)
self.reached_points = 0
self.closest = np.ones(len(points), dtype=np.int)
self.L = np.ones(len(points)) * np.inf
self.dv = np.zeros((len(points), 3))
self.vectors_sm = SharedMemory(name=vectors_UID)
self.tree_sm = SharedMemory(name=tree_UID)
self.nodes = as_numpy_arr((self.maxsize, 3), self.tree_sm)
self.vectors = as_numpy_arr((self.maxsize, 3), self.vectors_sm)
self.pipe = pipe
self.lock = lock
self.batch = None
async def object_info(self, object_id) -> ObjectInfo:
shm = SharedMemory(name=object_id)
if _is_windows:
return WinShmObjectInfo(size=shm.size,
object_id=object_id,
shm=shm)
else:
return ObjectInfo(size=shm.size, object_id=object_id)
def read_test(x, y, shm_name, shm_shape, shm_dtype):
# Locate the shared memory by its name
shm = SharedMemory(shm_name)
# Create the np.ndarray from the buffer of the shared memory
np_array = np.ndarray(shape=shm_shape, dtype=shm_dtype, buffer=shm.buf)
result = np_array[x][y]
print("RESULT:", result)
return result
def read_from_shared(x, y, shm_name, shm_shape, shm_dtype):
print("SWONEEE")
# Locate the shared memory by its name
shm = SharedMemory(shm_name)
# Create the np.recarray from the buffer of the shared memory
np_array = np.recarray(shape=shm_shape, dtype=shm_dtype, buf=shm.buf)
return np.nansum(np_array.val)
"""
def init_from_data_buffer(self, data_buffer):
self.buffer = data_buffer
self.shms = traverse_lists(
self.buffer, lambda b: [
SharedMemory(create=True, size=b.nbytes)
for _ in range(self.queue_size)
])
def getMM(size, create=True):
if isWindows:
from mmap import mmap
return mmap(-1, size, tagname=mmName)
else: # requires Python >= 3.8
from multiprocessing.shared_memory import SharedMemory
shm = SharedMemory(mmName, create, size)
return shm
def find_adv_batch(self, model: nn.Module, inputs: torch.Tensor,
inputs_adv_ref: torch.Tensor, labels: torch.Tensor,
epsilon: float, max_nr_adv):
self._start_workers()
with ensure_training_state(model, False):
model_outputs: torch.Tensor = model(inputs_adv_ref)
eval_model = model.cvt_to_eval()
correct_mask = torch.eq(model_outputs.argmax(dim=1), labels)
idx_remap = np.arange(inputs.shape[0], dtype=np.int32)
np.random.shuffle(idx_remap)
assert inputs.dtype == torch.float32
shm_size = BUFFER_COUNTER_SIZE + 4 * inputs.numel()
shm = SharedMemory(size=shm_size, create=True)
shm.buf[:BUFFER_COUNTER_SIZE] = b'\0' * BUFFER_COUNTER_SIZE
shm_name = shm.name
args = RemoteArgs(eval_model, inputs, inputs_adv_ref, labels,
correct_mask, epsilon, max_nr_adv, shm_name,
shm_size, idx_remap)
try:
return self._work(shm.buf, args)
except:
self.close()
raise
finally:
shm.close()
shm.unlink()
def init_shared_memory(self, name, shape, dtype, config):
log = config.logger
matrix = np.zeros(shape, dtype=dtype)
memory = SharedMemory(name, create=True, size=matrix.nbytes)
log.info(
f"Memory allocated for {name} with shape {shape} and dtype {dtype}. Overall size in memory: "
f"{matrix.nbytes/1024/1024:.2f} MBytes ({matrix.nbytes} bytes)")
del matrix
self._memories.append(memory)
def __init__(self, queue, out_queue, base_path, shm, cache_path=None, logging_queue=None):
super().__init__(name='FileWorker')
self.q = queue
self.o_q = out_queue
self.base_path = base_path
self.cache_path = cache_path if cache_path else os.path.join(base_path, '.cache')
self.shm = SharedMemory(name=shm)
self.log_level = logging.getLogger().level
self.logging_queue = logging_queue
def is_i(task: int, config: Config, embedding_memory: MemoryInfo,
distance_memory: MemoryInfo, lamb):
# Distance space
dist_mem = SharedMemory(distance_memory.name)
distance_matrix = np.ndarray(shape=distance_memory.shape,
buffer=dist_mem.buf)
return is_p(task, j_star(task, distance_matrix), config, embedding_memory,
lamb)
def animate(i, ax, line, rd, queue, a_queue, lines_poiis, constants):
if i < iterations - 1:
frame_i = i + 1
else:
frame_i = i % iterations
ax.set_title(str(frame_i))
start_time = time.perf_counter()
if not rd.finished_loading and not a_queue.empty():
data_shared_info = a_queue.get(block=True)
rd.data_shared = SharedMemory(data_shared_info['name'])
rd.data = np.ndarray(shape=data_shared_info['shape'],
dtype=data_shared_info['dtype'],
buffer=rd.data_shared.buf)
time0 = time.perf_counter() - start_time
start_time = time.perf_counter()
time1 = time.perf_counter() - start_time
start_time = time.perf_counter()
rd.finished_loading = True
a_queue.task_done()
a_queue.join()
time2 = time.perf_counter() - start_time
print("Put data!")
# clear_process = Process(target=clear_queue, args=(queue,))
# clear_process.start()
# rd.clearer = clear_process
# print(f"{time0:0.8f}" + " " + f"{time1:0.8f}" + " " + f"{time2:0.8f}")
start_time = time.perf_counter()
if not rd.finished_loading and frame_i % chunksize == 0:
rd.set_reading_chunk(queue.get(block=True))
queue.task_done()
time4 = time.perf_counter() - start_time
start_time = time.perf_counter()
row = rd.get_row(frame_i)
row = row.reshape(int(len(row) / 2), 2).transpose()
time5 = time.perf_counter() - start_time
# print(f"{time4:0.8f}" + " " + f"{time5:0.8f}")
t = frame_i * constants['dt']
T = constants['omega'] * t
o_x = constants['radius_gal'] * np.cos(T)
o_y = constants['radius_gal'] * np.sin(T)
lines_poiis['o_line'].set_xdata([0, o_x])
lines_poiis['o_line'].set_ydata([0, o_y])
rd.update_poii_data(row)
x_poiis = rd.poiis_x
y_poiis = rd.poiis_y
s_i = min(len(rd.poiis_x[0]), 300)
for i, poii_line in enumerate(lines_poiis['lines']):
poii_line.set_xdata(x_poiis[i][-s_i:])
poii_line.set_ydata(y_poiis[i][-s_i:])
line.set_xdata(row[0])
line.set_ydata(row[1])
def score(config: Config,
embedding_memory: MemoryInfo,
distance_memory: MemoryInfo,
lamb=5):
"""
Calculating interpretability scores
:param config: Config object
:param embedding_memory: Memory info about the embedding
:param distance_memory: Memory info about the distance matrix
:param proc: Number of processes to use
:param lamb: the number of lambda to compute from 1
:return:
"""
IS_i = []
number_of_processes = config.project.processes
pool = multiprocessing.Pool(processes=number_of_processes)
# Results
r = np.zeros([lamb, embedding_memory.shape[-1]], dtype=np.float)
results_name = f"{config.memory_prefix}_lambdas_per_dim"
results_mem = SharedMemory(name=results_name, create=True, size=r.nbytes)
buf = np.ndarray(r.shape, dtype=r.dtype, buffer=results_mem.buf)
buf[:, :] = r[:, :]
del r
relaxation_memory = MemoryInfo()
relaxation_memory.name = results_name
relaxation_memory.shape = buf.shape
task_manager = Manager()
task_queue = task_manager.Queue()
progress_queue = task_manager.Queue()
for i in range(config.semantic_categories.categories.i2c.__len__()):
task_queue.put(i)
inputs = []
for i in range(number_of_processes):
inputs.append([
config, embedding_memory, distance_memory, task_queue,
progress_queue, relaxation_memory, lamb
])
progress = Process(target=_progress_bar,
args=(progress_queue, task_queue.qsize()))
progress.start()
with pool as p:
_ = p.starmap(score_dist, inputs)
progress.join()
res = np.mean(buf, axis=1)
IS_i = [res[i] for i in range(res.shape[0])]
return IS_i
