def generate_embedding_from_hdf5(
data_gen: Iterable,
output_path: pathlib.Path,
output_dim: int,
count: int,
):
file_exists = output_path.is_file()
with h5py.File(str(output_path), "a") as f:
if not file_exists:
embedding_dset = f.create_dataset("embedding", (count, output_dim),
dtype="f",
chunks=True)
external_id_dset = f.create_dataset("external_id", (count, ),
dtype="i",
chunks=True)
offset = 0
else:
offset = get_offset(f)
embedding_dset = f["embedding"]
external_id_dset = f["external_id"]
print("Offset: {}".format(offset))
for (embeddings_batch, external_id_batch) in data_gen:
slicing = slice(offset, offset + len(embeddings_batch))
embedding_dset[slicing] = embeddings_batch
external_id_dset[slicing] = external_id_batch
offset += len(embeddings_batch)
def patterns_to_csv(patterns, midi_score, h):
"""Formats the patterns into the csv format.
Parameters
----------
pattersn : list
List of patterns with its occurrences.
midi_score : list
The score of the piece (read from CSV).
h : float
Hop size of the ssm.
Returns
-------
csv_patterns : list
List of the patterns in the csv format to be analyzed by MIREX.
"""
offset = np.abs(int(utils.get_offset(midi_score) / float(h)))
csv_patterns = []
for p in patterns:
new_p = []
for occ in p:
start = occ[2] * h + offset - 1
end = occ[3] * h + offset + 1 # Add the diff offset
csv_occ = occurrence_to_csv(start, end, midi_score)
if csv_occ != []:
new_p.append(csv_occ)
if new_p != [] and len(new_p) >= 2:
csv_patterns.append(new_p)
return csv_patterns
def run_bot():
TOKEN = read_file(TOKEN_FILE)
KEYWORD = read_file(KEYWORD_FILE)
REPLY = read_file(REPLY_FILE)
OFFSET = read_file(OFFSET_FILE)
exclusion_list = read_file(EXCLUSION_FILE)
bot = create_bot(TOKEN)
while True:
updates = fetch_updates(bot, OFFSET)
if updates is not None:
process_updates(updates, bot, KEYWORD, REPLY, exclusion_list)
OFFSET = get_offset(updates)
create_file(OFFSET_FILE, OFFSET)
else:
time.sleep(1)
def get_odom_delta(self, pose):
"""Calculates the delta of the odometry position since the last time called.
Returns None the first time, and a 3x1 state matrix thereafter.
"""
# Convert the pose into matrix form.
y_odom = column_vector(pose.x, pose.y, pose.theta)
# None is the fallback in case we can't get an actual delta.
odom_delta = None
if self.odom_state is not None:
# Transform the sensor state into the map frame.
y = coord_transform(y_odom, self.odom_origin)
# Calculate the change odom state, in map coords (delta y).
odom_delta = y - coord_transform(self.odom_state, self.odom_origin)
# Update the stored odom state.
self.odom_state = y_odom
# Get the odom frame origin in the map frame and store it for next time.
self.odom_origin = get_offset(self.state, self.odom_state)
return odom_delta
def get_odom_delta(self, pose):
"""Calculates the delta of the odometry position since the last time called.
Returns None the first time, and a 3x1 state matrix thereafter.
"""
# Convert the pose into matrix form.
y_odom = column_vector(pose.x, pose.y, pose.theta)
# None is the fallback in case we can't get an actual delta.
odom_delta = None
if self.odom_state is not None:
# Transform the sensor state into the map frame.
y = coord_transform(y_odom, self.odom_origin)
# Calculate the change odom state, in map coords (delta y).
odom_delta = y - coord_transform(self.odom_state, self.odom_origin)
# Update the stored odom state.
self.odom_state = y_odom
# Get the odom frame origin in the map frame and store it for next time.
self.odom_origin = get_offset(self.state, self.odom_state)
return odom_delta
def get_odom_delta(self, pose):
# odom_state is the previous x, y, theta of odom_pose
# y_odom is the current x, y, theta of odom_pose
# odom_origin kind keeps track of all the odom changes till date
y_odom = column_vector(pose.x, pose.y, pose.theta)
odom_delta = None
if self.odom_state is not None:
# y = y_odom * rot(odom_origin.theta) + odom_origin
y = coord_transform(y_odom, self.odom_origin)
# current change - previous change
odom_delta = y - coord_transform(self.odom_state, self.odom_origin)
self.odom_state = y_odom
# these two should be exactly the same. Using y_odom to
# differentiate between previous and new odom values
#self.odom_origin = get_offset(self.state, self.odom_state)
self.odom_origin = get_offset(self.state, y_odom)
return odom_delta
def get_odom_delta(self, pose): # odom_state is the previous x, y, theta of odom_pose # y_odom is the current x, y, theta of odom_pose # odom_origin kind keeps track of all the odom changes till date y_odom = column_vector(pose.x, pose.y, pose.theta) odom_delta = None if self.odom_state is not None: # y = y_odom * rot(odom_origin.theta) + odom_origin y = coord_transform(y_odom, self.odom_origin) # current change - previous change odom_delta = y - coord_transform(self.odom_state, self.odom_origin) self.odom_state = y_odom # these two should be exactly the same. Using y_odom to # differentiate between previous and new odom values #self.odom_origin = get_offset(self.state, self.odom_state) self.odom_origin = get_offset(self.state, y_odom) return odom_delta
def __init__(self,
list_file,
patch_shape=PATCH_SHAPE,
kernels=KERNELS,
scale_factor=SCALE_FACTOR,
root='',
split='valid',
sample_size=20):
with open(list_file) as f:
names = f.read().splitlines()
names = [os.path.join(root, name) for name in names]
self.root = root
self.names = names
self.split = split
self.sample_size = sample_size
self.receptive_field = get_receptive_field(kernels)
self.patch_shape = np.array(patch_shape)
self.scale_factor = np.array(scale_factor)
self.sub_patch_shape = get_sub_patch_shape(self.patch_shape,
self.receptive_field, self.scale_factor)
self.sub_off = get_offset(self.scale_factor, self.receptive_field)
self.modalities = ('Flair', 'T1c', 'T1', 'T2')
self.C = len(self.modalities)
def save_hdf5(
output_file: pathlib.Path,
data_iter: Iterable[Tuple[str, int, np.ndarray, Tuple[int, int],
List[float], float, int]],
count: int,
size: int,
batch_size: int = 256,
compression: Optional[str] = None,
):
file_exists = output_file.is_file()
with h5py.File(str(output_file), "a") as f:
if not file_exists:
image_dset = f.create_dataset(
"image",
(count, size, size, 3),
dtype="uint8",
compression=compression,
chunks=(2048, size, size, 3),
)
barcode_dset = f.create_dataset(
"barcode",
(count, ),
dtype=h5py.string_dtype(),
compression=compression,
chunks=(2048, ),
)
image_id_dset = f.create_dataset(
"image_id",
(count, ),
dtype="i",
compression=compression,
chunks=(2048, ),
)
resolution_dset = f.create_dataset(
"resolution",
(count, 2),
dtype="i",
compression=compression,
chunks=(2048, 2),
)
bounding_box_dset = f.create_dataset(
"bounding_box",
(count, 4),
dtype="f",
compression=compression,
chunks=(2048, 4),
)
confidence_dset = f.create_dataset(
"confidence",
(count, ),
dtype="f",
compression=compression,
chunks=(2048, ),
)
external_id_dset = f.create_dataset(
"external_id",
(count, ),
dtype="i",
compression=compression,
chunks=(2048, ),
)
offset = 0
else:
offset = get_offset(f)
image_dset = f["image"]
barcode_dset = f["barcode"]
image_id_dset = f["image_id"]
resolution_dset = f["resolution"]
bounding_box_dset = f["bounding_box"]
confidence_dset = f["confidence"]
external_id_dset = f["external_id"]
print("Offset: {}".format(offset))
for batch in chunked(data_iter, batch_size):
barcode_batch = np.array([x[0] for x in batch])
image_id_batch = np.array([x[1] for x in batch], dtype="i")
image_batch = np.stack([x[2] for x in batch], axis=0)
resolution_batch = np.array([list(x[3]) for x in batch])
bounding_box_batch = np.array([list(x[4]) for x in batch])
confidence_batch = np.array([x[5] for x in batch])
external_id_batch = np.array([x[6] for x in batch])
slicing = slice(offset, offset + len(batch))
barcode_dset[slicing] = barcode_batch
image_id_dset[slicing] = image_id_batch
image_dset[slicing] = image_batch
resolution_dset[slicing] = resolution_batch
bounding_box_dset[slicing] = bounding_box_batch
confidence_dset[slicing] = confidence_batch
external_id_dset[slicing] = external_id_batch
offset += len(batch)
