def test_find_position(self):
data = [2, 6, -24, 65, 46, 5, -10, 9]
to_find = [54, 12, 5, -10, 45, 6, 2]
positions = find_positions(data, to_find)
self.assertEqual(positions.tolist(), [-1, -1, 5, 6, -1, 1, 0])
def _find_place_for_segments_in_memory(self, segments: Sequence, segment_lengths: Sequence) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
"""
1. Find known segments
2. Find empty spaces with fitting length
3. Find empty spaces with bigger length
4. Amend remaining segments
:param segments:
:param segment_lengths:
:return:
"""
segment_hashes = np.fromiter((hash(segment) for segment in segments), count=len(segments), dtype=np.int64)
waveform_to_segment = find_positions(self._segment_hashes, segment_hashes)
# separate into known and unknown
unknown = (waveform_to_segment == -1)
known = ~unknown
known_pos_in_memory = waveform_to_segment[known]
assert len(known_pos_in_memory) == 0 or np.all(self._segment_hashes[known_pos_in_memory] == segment_hashes[known])
new_reference_counter = self._segment_references.copy()
new_reference_counter[known_pos_in_memory] += 1
to_upload_size = np.sum(segment_lengths[unknown] + 16)
free_points_in_total = self.total_capacity - np.sum(self._segment_capacity[self._segment_references > 0])
if free_points_in_total < to_upload_size:
raise MemoryError('Not enough free memory',
free_points_in_total,
to_upload_size,
self._free_points_in_total)
to_amend = cast(np.ndarray, unknown)
to_insert = np.full(len(segments), fill_value=-1, dtype=np.int64)
reserved_indices = np.flatnonzero(new_reference_counter > 0)
first_free = reserved_indices[-1] + 1 if len(reserved_indices) else 0
free_segments = new_reference_counter[:first_free] == 0
free_segment_count = np.sum(free_segments)
# look for a free segment place with the same length
for segment_idx in np.flatnonzero(to_amend):
if free_segment_count == 0:
break
pos_of_same_length = np.logical_and(free_segments, segment_lengths[segment_idx] == self._segment_capacity[:first_free])
idx_same_length = np.argmax(pos_of_same_length)
if pos_of_same_length[idx_same_length]:
free_segments[idx_same_length] = False
free_segment_count -= 1
to_amend[segment_idx] = False
to_insert[segment_idx] = idx_same_length
# try to find places that are larger than the segments to fit in starting with the large segments and large
# free spaces
segment_indices = np.flatnonzero(to_amend)[np.argsort(segment_lengths[to_amend])[::-1]]
capacities = self._segment_capacity[:first_free]
for segment_idx in segment_indices:
free_capacities = capacities[free_segments]
free_segments_indices = np.flatnonzero(free_segments)[np.argsort(free_capacities)[::-1]]
if len(free_segments_indices) == 0:
break
fitting_segment = np.argmax((free_capacities >= segment_lengths[segment_idx])[::-1])
fitting_segment = free_segments_indices[fitting_segment]
if self._segment_capacity[fitting_segment] >= segment_lengths[segment_idx]:
free_segments[fitting_segment] = False
to_amend[segment_idx] = False
to_insert[segment_idx] = fitting_segment
free_points_at_end = self.total_capacity - np.sum(self._segment_capacity[:first_free])
if np.sum(segment_lengths[to_amend] + 16) > free_points_at_end:
raise MemoryError('Fragmentation does not allow upload.',
np.sum(segment_lengths[to_amend] + 16),
free_points_at_end,
self._free_points_at_end)
return waveform_to_segment, to_amend, to_insert