def __init__(
self, trial_ids, rgb_attributes_dir, rgb_data_dir, imu_attributes_dir, imu_data_dir,
device=None, modalities=None):
self.trial_ids = trial_ids
self.metadata = utils.loadMetadata(rgb_data_dir, rows=trial_ids)
self.vocab = utils.loadVariable('vocab', rgb_attributes_dir)
self.rgb_attributes_dir = rgb_attributes_dir
self.rgb_data_dir = rgb_data_dir
self.imu_attributes_dir = imu_attributes_dir
self.imu_data_dir = imu_data_dir
self.device = device
self.modalities = modalities
def main(out_dir=None, data_dir=None, labels_dir=None):
out_dir = os.path.expanduser(out_dir)
data_dir = os.path.expanduser(data_dir)
labels_dir = os.path.expanduser(labels_dir)
logger = utils.setupRootLogger(filename=os.path.join(out_dir, 'log.txt'))
fig_dir = os.path.join(out_dir, 'figures')
if not os.path.exists(fig_dir):
os.makedirs(fig_dir)
out_data_dir = os.path.join(out_dir, 'data')
if not os.path.exists(out_data_dir):
os.makedirs(out_data_dir)
filenames = [
utils.stripExtension(fn)
for fn in glob.glob(os.path.join(labels_dir, '*.csv'))
]
metadata = utils.loadMetadata(data_dir)
metadata['seq_id'] = metadata.index
metadata = metadata.set_index(
'dir_name', drop=False).loc[filenames].set_index('seq_id')
seq_ids = np.sort(metadata.index.to_numpy())
logger.info(f"Loaded {len(seq_ids)} sequences from {labels_dir}")
vocab = []
for i, seq_id in enumerate(seq_ids):
seq_id_str = f"seq={seq_id}"
seq_dir_name = metadata['dir_name'].loc[seq_id]
labels_fn = os.path.join(labels_dir, f'{seq_dir_name}.csv')
event_labels = utils.loadVariable(f'{seq_id_str}_labels', data_dir)
assembly_actions = pd.read_csv(labels_fn)
label_seq = parseActions(assembly_actions, event_labels.shape[0],
vocab)
utils.saveVariable(label_seq, f'{seq_id_str}_label-seq', out_data_dir)
plotLabels(os.path.join(fig_dir, f'{seq_id_str}_labels.png'),
label_seq)
writeLabels(os.path.join(fig_dir, f'{seq_id_str}_labels.csv'),
label_seq, vocab)
utils.saveMetadata(metadata, out_data_dir)
utils.saveVariable(vocab, 'vocab', out_data_dir)
def __init__(self,
actions_dir,
parts_dir,
events_dir,
edges_dir,
vocab=None,
modalities=('actions', 'parts', 'edges'),
labels='edges',
prefix='seq=',
feature_fn_format='score-seq',
label_fn_format='true-label-seq'):
self.modalities = modalities
self.labels = labels
self.actions_dir = actions_dir
self.parts_dir = parts_dir
self.events_dir = events_dir
self.edges_dir = edges_dir
self.data_dirs = {
'actions': self.actions_dir,
'parts': self.parts_dir,
'events': self.events_dir,
'edges': self.edges_dir
}
labels_dir = self.data_dirs[self.labels]
self.trial_ids = utils.getUniqueIds(
labels_dir,
prefix='trial=' if self.labels == 'edges' else prefix,
suffix=f'{label_fn_format}.*',
to_array=True)
if vocab is None:
vocab = utils.loadVariable('vocab', labels_dir)
self.vocab = vocab
self.metadata = utils.loadMetadata(labels_dir, rows=self.trial_ids)
self.prefix = prefix
self.feature_fn_format = feature_fn_format
self.label_fn_format = label_fn_format
def main(
out_dir=None, data_dir=None, use_vid_ids_from=None,
output_data=None, magnitude_centering=None, resting_from_gt=None,
remove_before_first_touch=None, include_signals=None, fig_type=None):
data_dir = os.path.expanduser(data_dir)
out_dir = os.path.expanduser(out_dir)
logger = utils.setupRootLogger(filename=os.path.join(out_dir, 'log.txt'))
logger.info(f"Reading from: {data_dir}")
logger.info(f"Writing to: {out_dir}")
fig_dir = os.path.join(out_dir, 'figures')
if not os.path.exists(fig_dir):
os.makedirs(fig_dir)
out_data_dir = os.path.join(out_dir, 'data')
if not os.path.exists(out_data_dir):
os.makedirs(out_data_dir)
def loadAll(seq_ids, var_name, from_dir=data_dir, prefix='trial='):
all_data = tuple(
utils.loadVariable(f"{prefix}{seq_id}_{var_name}", from_dir)
for seq_id in seq_ids
)
return all_data
def saveVariable(var, var_name, to_dir=out_data_dir):
utils.saveVariable(var, var_name, to_dir)
if fig_type is None:
fig_type = 'multi'
# Load data
if use_vid_ids_from is None:
trial_ids = utils.getUniqueIds(data_dir, prefix='trial=', to_array=True)
else:
use_vid_ids_from = os.path.expanduser(use_vid_ids_from)
trial_ids = utils.getUniqueIds(use_vid_ids_from, prefix='trial-', to_array=True)
accel_seqs = loadAll(trial_ids, 'accel-samples.pkl')
gyro_seqs = loadAll(trial_ids, 'gyro-samples.pkl')
action_seqs = loadAll(trial_ids, 'action-seq.pkl')
rgb_timestamp_seqs = loadAll(trial_ids, 'rgb-frame-timestamp-seq.pkl')
def validate_imu(seqs):
def is_valid(d):
return not any(np.isnan(x).any() for x in d.values())
return np.array([is_valid(d) for d in seqs])
imu_is_valid = validate_imu(accel_seqs) & validate_imu(gyro_seqs)
logger.info(
f"Ignoring {(~imu_is_valid).sum()} IMU sequences with NaN-valued samples "
f"(of {len(imu_is_valid)} total)"
)
def chooseValid(seq):
return tuple(x for x, is_valid in zip(seq, imu_is_valid) if is_valid)
trial_ids = np.array(list(chooseValid(trial_ids)))
accel_seqs = chooseValid(accel_seqs)
gyro_seqs = chooseValid(gyro_seqs)
action_seqs = chooseValid(action_seqs)
rgb_timestamp_seqs = chooseValid(rgb_timestamp_seqs)
vocab = []
metadata = utils.loadMetadata(data_dir, rows=trial_ids)
utils.saveMetadata(metadata, out_data_dir)
utils.saveVariable(vocab, 'vocab', out_data_dir)
def norm(x):
norm = np.linalg.norm(imu.getImuSamples(x), axis=1)[:, None]
return norm
accel_mag_seqs = tuple(map(lambda x: dictToArray(x, transform=norm), accel_seqs))
gyro_mag_seqs = tuple(map(lambda x: dictToArray(x, transform=norm), gyro_seqs))
imu_timestamp_seqs = utils.batchProcess(makeTimestamps, accel_seqs, gyro_seqs)
if remove_before_first_touch:
before_first_touch_seqs = utils.batchProcess(
beforeFirstTouch, action_seqs, rgb_timestamp_seqs, imu_timestamp_seqs
)
num_ignored = sum(b is None for b in before_first_touch_seqs)
logger.info(
f"Ignoring {num_ignored} sequences without first-touch annotations "
f"(of {len(before_first_touch_seqs)} total)"
)
trials_missing_first_touch = [
i for b, i in zip(before_first_touch_seqs, trial_ids)
if b is None
]
logger.info(f"Trials without first touch: {trials_missing_first_touch}")
def clip(signal, bool_array):
return signal[~bool_array, ...]
accel_mag_seqs = tuple(
clip(signal, b) for signal, b in zip(accel_mag_seqs, before_first_touch_seqs)
if b is not None
)
gyro_mag_seqs = tuple(
clip(signal, b) for signal, b in zip(gyro_mag_seqs, before_first_touch_seqs)
if b is not None
)
imu_timestamp_seqs = tuple(
clip(signal, b) for signal, b in zip(imu_timestamp_seqs, before_first_touch_seqs)
if b is not None
)
trial_ids = tuple(
x for x, b in zip(trial_ids, before_first_touch_seqs)
if b is not None
)
action_seqs = tuple(
x for x, b in zip(action_seqs, before_first_touch_seqs)
if b is not None
)
rgb_timestamp_seqs = tuple(
x for x, b in zip(rgb_timestamp_seqs, before_first_touch_seqs)
if b is not None
)
assembly_seqs = utils.batchProcess(
parseActions,
action_seqs, rgb_timestamp_seqs, imu_timestamp_seqs
)
if output_data == 'components':
accel_feat_seqs = accel_mag_seqs
gyro_feat_seqs = gyro_mag_seqs
unique_components = {frozenset(): 0}
imu_label_seqs = zip(
*tuple(
labels.componentLabels(*args, unique_components)
for args in zip(action_seqs, rgb_timestamp_seqs, imu_timestamp_seqs)
)
)
saveVariable(unique_components, 'unique_components')
elif output_data == 'pairwise components':
imu_label_seqs = utils.batchProcess(
labels.pairwiseComponentLabels, assembly_seqs,
static_kwargs={'lower_tri_only': True, 'include_action_labels': False}
)
accel_feat_seqs = tuple(map(imu.pairwiseFeats, accel_mag_seqs))
gyro_feat_seqs = tuple(map(imu.pairwiseFeats, gyro_mag_seqs))
else:
raise AssertionError()
signals = {'accel': accel_feat_seqs, 'gyro': gyro_feat_seqs}
if include_signals is None:
include_signals = tuple(signals.keys())
signals = tuple(signals[key] for key in include_signals)
imu_feature_seqs = tuple(np.stack(x, axis=-1).squeeze(axis=-1) for x in zip(*signals))
video_seqs = tuple(zip(imu_feature_seqs, imu_label_seqs, trial_ids))
imu.plot_prediction_eg(video_seqs, fig_dir, fig_type=fig_type, output_data=output_data)
video_seqs = tuple(
zip(assembly_seqs, imu_feature_seqs, imu_timestamp_seqs, imu_label_seqs, trial_ids)
)
for assembly_seq, feature_seq, timestamp_seq, label_seq, trial_id in video_seqs:
id_string = f"trial={trial_id}"
saveVariable(assembly_seq, f'{id_string}_assembly-seq')
saveVariable(feature_seq, f'{id_string}_feature-seq')
saveVariable(timestamp_seq, f'{id_string}_timestamp-seq')
saveVariable(label_seq, f'{id_string}_label-seq')
def main(
out_dir=None, data_dir=None, results_file=None, cv_file=None,
take_log=False, col_format=None, win_params={}, slowfast_csv_params={}):
out_dir = os.path.expanduser(out_dir)
data_dir = os.path.expanduser(data_dir)
results_file = os.path.expanduser(results_file)
cv_file = os.path.expanduser(cv_file)
logger = utils.setupRootLogger(filename=os.path.join(out_dir, 'log.txt'))
fig_dir = os.path.join(out_dir, 'figures')
if not os.path.exists(fig_dir):
os.makedirs(fig_dir)
out_data_dir = os.path.join(out_dir, 'data')
if not os.path.exists(out_data_dir):
os.makedirs(out_data_dir)
vocab = utils.loadVariable('vocab', data_dir)
metadata = utils.loadMetadata(data_dir)
slowfast_labels = pd.read_csv(
cv_file, keep_default_na=False, index_col=0,
**slowfast_csv_params
)
seg_ids = slowfast_labels.index.to_numpy()
vid_names = slowfast_labels['video_name'].unique().tolist()
metadata['seq_id'] = metadata.index
vid_ids = metadata.set_index('dir_name').loc[vid_names].set_index('seq_id').index
metadata = metadata.drop('seq_id', axis=1)
with open(results_file, 'rb') as file_:
model_probs, gt_labels = pickle.load(file_)
model_probs = model_probs.numpy()
gt_labels = gt_labels.numpy()
if len(model_probs) != len(seg_ids):
err_str = f"{len(model_probs)} segment scores != {slowfast_labels.shape[0]} CSV rows"
raise AssertionError(err_str)
logger.info(f"Loaded {len(seg_ids)} segments, {len(vid_ids)} videos")
for vid_id, vid_name in zip(vid_ids, vid_names):
matches_video = (slowfast_labels['video_name'] == vid_name).to_numpy()
win_labels = gt_labels[matches_video]
win_probs = model_probs[matches_video, :]
if win_labels.shape == win_probs.shape:
win_preds = (win_probs > 0.5).astype(int)
else:
win_preds = win_probs.argmax(axis=1)
if take_log:
win_probs = np.log(win_probs)
seq_id_str = f"seq={vid_id}"
utils.saveVariable(win_probs, f'{seq_id_str}_score-seq', out_data_dir)
utils.saveVariable(win_labels, f'{seq_id_str}_true-label-seq', out_data_dir)
utils.saveVariable(win_preds, f'{seq_id_str}_pred-label-seq', out_data_dir)
utils.plot_array(
win_probs.T, (win_labels.T, win_preds.T), ('true', 'pred'),
tick_names=vocab,
fn=os.path.join(fig_dir, f"{seq_id_str}.png"),
subplot_width=12, subplot_height=5
)
utils.saveVariable(vocab, 'vocab', out_data_dir)
utils.saveMetadata(metadata, out_data_dir)