def process_hdf5(flags, h5_data, base_dir, mouse, input_types, views, out):
# create directory
# data_dir = os.path.join(
# "/groups/branson/bransonlab/kwaki/data/features/finetune2_i3d",
# mouse
# )
# data_dir = os.path.join(
# "/groups/branson/bransonlab/kwaki/data/features/finetune_odas_unfroze/",
# mouse
# )
# "/groups/branson/bransonlab/kwaki/data/features/finetune_odas_unfroze_reweight/",
data_dir = os.path.join(
"/groups/branson/bransonlab/kwaki/data/features/finetune_odas_unfroze_reweight",
mouse
)
# # setup the flow.
# print("\t\tflow")
# mouse_dir = os.path.join(base_dir, mouse, "all", input_types[0])
# opts = setup_opts(flags, mouse_dir)
# paths.setup_output_space(opts)
# compute_outputs(opts, h5_data, [input_types[0]], views, data_dir, out)
# setup the rgb.
print("\t\trgb")
mouse_dir = os.path.join(base_dir, mouse, "all", input_types[0])
opts = setup_opts(flags, mouse_dir)
paths.setup_output_space(opts)
compute_outputs(opts, h5_data, [input_types[0]], views, data_dir, out)
def main(argv):
opts = _setup_opts(argv)
paths.setup_output_space(opts)
if opts["flags"].cuda_device != -1:
torch.cuda.set_device(opts["flags"].cuda_device)
with h5py.File(opts["flags"].train_file, "r") as train_data:
with h5py.File(opts["flags"].test_file, "r") as test_data:
with h5py.File(opts["flags"].valid_file, "r") as valid_data:
if DEBUG:
train_data = valid_data
test_data = valid_data
sequences_helper.copy_templates(opts, train_data, "train",
g_label_names)
sequences_helper.copy_templates(opts, test_data, "test",
g_label_names)
sequences_helper.copy_templates(opts, valid_data, "valid",
g_label_names)
sampler = HantmanVideoFrameSampler(
opts["rng"],
train_data,
opts["flags"].video_dir,
opts["flags"].hantman_mini_batch,
frames=opts["flags"].frames,
use_pool=True,
gpu_id=opts["flags"].cuda_device)
label_weight = _get_label_weight(opts, train_data)
# import pdb; pdb.set_trace()
train_eval = HantmanVideoSampler(
opts["rng"],
train_data,
opts["flags"].video_dir,
use_pool=True,
gpu_id=opts["flags"].cuda_device)
test_eval = HantmanVideoSampler(
opts["rng"],
test_data,
opts["flags"].video_dir,
use_pool=True,
gpu_id=opts["flags"].cuda_device)
valid_eval = HantmanVideoSampler(
opts["rng"],
valid_data,
opts["flags"].video_dir,
use_pool=True,
gpu_id=opts["flags"].cuda_device)
network, optimizer, criterion = _init_network(
opts, label_weight)
# import pdb; pdb.set_trace()
_train_network(opts, network, optimizer, criterion, sampler,
train_eval, test_eval, valid_eval)
def main(argv):
opts = _setup_opts(argv)
paths.setup_output_space(opts)
if not os.path.exists(opts["flags"].feature_dir):
os.makedirs(opts["flags"].feature_dir)
exp_dir = os.path.join(os.path.dirname(opts["flags"].train_file),
"exps",
os.path.basename(opts["flags"].feature_dir))
if opts["flags"].cuda_device != -1:
torch.cuda.set_device(opts["flags"].cuda_device)
# load data
with h5py.File(opts["flags"].train_file, "a") as train_data:
with h5py.File(opts["flags"].test_file, "a") as test_data:
with h5py.File(opts["flags"].valid_file, "a") as valid_data:
if DEBUG:
train_data = valid_data
test_data = valid_data
sampler = None
label_weight = _get_label_weight(opts, train_data)
train_eval = HantmanVideoSampler(
None,
train_data,
opts["flags"].video_dir,
use_pool=True,
gpu_id=opts["flags"].cuda_device)
test_eval = HantmanVideoSampler(
None,
test_data,
opts["flags"].video_dir,
use_pool=True,
gpu_id=opts["flags"].cuda_device)
valid_eval = HantmanVideoSampler(
None,
valid_data,
opts["flags"].video_dir,
use_pool=True,
gpu_id=opts["flags"].cuda_device)
sequences_helper.copy_templates(opts, train_data, "train",
g_label_names)
sequences_helper.copy_templates(opts, test_data, "test",
g_label_names)
sequences_helper.copy_templates(opts, valid_data, "valid",
g_label_names)
network, optimizer, criterion = _init_model(opts, label_weight)
_train_network(opts, network, optimizer, criterion, sampler,
train_eval, test_eval, valid_eval)
def main(argv):
print(argv)
opts = _setup_opts(argv)
paths.setup_output_space(opts)
if opts["flags"].cuda_device != -1:
torch.cuda.set_device(opts["flags"].cuda_device)
full_tic = time.time()
with h5py.File(opts["flags"].train_file, "r") as train_data:
with h5py.File(opts["flags"].test_file, "r") as test_data:
if opts["flags"].valid_file is not None:
with h5py.File(opts["flags"].valid_file, "r") as valid_data:
run_training(opts, train_data, test_data, valid_data)
else:
run_training(opts, train_data, test_data, None)
print("Training took: %d\n" % (time.time() - full_tic))
def main(argv):
# setup options
opts = _setup_opts(argv)
paths.setup_output_space(opts)
if opts["flags"].cuda_device != -1:
torch.cuda.set_device(opts["flags"].cuda_device)
# load data
# timing here?
tic = time.time()
data_files = _load_hdfs(opts)
# setup the templates for plotting
_setup_templates(opts, data_files, g_label_names)
samplers = _setup_samplers(opts, data_files)
network, optimizer = _init_network(opts, samplers)
_train_network(opts, network, optimizer, samplers)
import pdb; pdb.set_trace()
print(time.time() - tic)
# close hdf5s
for data_file in data_files:
data_file.close()
def process_hdf5(flags, h5_data, base_dir, mouse, input_types, views, out):
# create directory
data_dir = os.path.join(
"/groups/branson/bransonlab/kwaki/data/features/finetune2_i3d", mouse)
# setup the flow.
print("\t\tflow")
mouse_dir = os.path.join(base_dir, mouse, "all", input_types[0])
opts = setup_opts(flags, mouse_dir)
paths.setup_output_space(opts)
compute_outputs(opts, h5_data, [input_types[0]], views, data_dir, out)
# setup the rgb.
print("\t\trgb")
mouse_dir = os.path.join(base_dir, mouse, "all", input_types[1])
opts = setup_opts(flags, mouse_dir)
paths.setup_output_space(opts)
compute_outputs(opts, h5_data, [input_types[1]], views, data_dir, out)
# setup the both. in this case
print("\t\tboth")
mouse_dir = os.path.join(base_dir, mouse, "all", "feedforward")
opts = setup_opts(flags, mouse_dir)
paths.setup_output_space(opts)
compute_outputs(opts, h5_data, input_types, views, data_dir, out)
def main(argv):
opts = _setup_opts(sys.argv)
paths.setup_output_space(opts)
if opts["flags"].cuda_device != -1:
torch.cuda.set_device(opts["flags"].cuda_device)
with h5py.File(opts["flags"].train_file, "r") as train_data:
with h5py.File(opts["flags"].test_file, "r") as test_data:
_copy_templates(opts, train_data, test_data)
print("done")
print("computing means")
tic = time.time()
temp_sampler = VideoSampler(opts["rng"],
train_data,
opts["flags"].video_dir,
seq_len=2000,
use_pool=False,
gpu_id=opts["flags"].cuda_device)
temp_sampler.reset()
normalize = compute_means(opts, train_data, temp_sampler)
print(time.time() - tic)
sampler = VideoFrameSampler(opts["rng"],
train_data,
opts["flags"].video_dir,
opts["flags"].hantman_mini_batch,
frames=opts["flags"].frames,
use_pool=False,
gpu_id=opts["flags"].cuda_device,
normalize=normalize)
# frames=range(-10, 11), use_pool=False,
# gpu_id=opts["flags"].cuda_device, normalize=normalize)
train_eval = VideoSampler(opts["rng"],
train_data,
opts["flags"].video_dir,
seq_len=-1,
use_pool=False,
gpu_id=opts["flags"].cuda_device,
normalize=normalize)
test_eval = VideoSampler(opts["rng"],
test_data,
opts["flags"].video_dir,
seq_len=-1,
use_pool=False,
gpu_id=opts["flags"].cuda_device,
normalize=normalize)
network, optimizer, criterion = _init_network(opts)
# batch = sampler.get_minibatch()
# import pdb; pdb.set_trace()
tic = time.time()
# network(Variable(batch[0], requires_grad=True))
_train_network(opts, network, optimizer, criterion, train_data,
test_data, sampler, train_eval, test_eval)
print(time.time() - tic)
import pdb
pdb.set_trace()
print("moo")
else:
for key in opts["flags"].feat_keys:
temp_feat = train_data["exps"][exp_names[0]][key].value
mean = np.zeros((temp_feat.shape[2], ))
std = np.ones((temp_feat.shape[2], ))
means.append(mean)
stds.append(std)
return means, stds
if __name__ == "__main__":
print(sys.argv)
opts = _setup_opts(sys.argv)
paths.setup_output_space(opts)
if opts["flags"].cuda_device != -1:
torch.cuda.set_device(opts["flags"].cuda_device)
# load data
# try:
tic = time.time()
with h5py.File(opts["flags"].train_file, "r") as train_data:
with h5py.File(opts["flags"].test_file, "r") as test_data:
with h5py.File(opts["flags"].val_file, "r") as val_data:
_copy_templates(opts, train_data, test_data, val_data)
label_weight = _get_label_weight(train_data)
label_mat = np.tile(
label_weight,
(opts["flags"].seq_len,
opts["flags"].hantman_mini_batch, 1)).astype('float32')
