def Euroc(root, train_sequence, test_sequence, split, stride, transform):
train_seq = [
'V2_01_easy', 'MH_02_easy', 'V1_03_difficult', 'V1_01_easy',
'V1_02_medium', 'V2_02_medium', 'MH_04_difficult', 'MH_03_medium'
]
test_seq = ['MH_05_difficult', 'V2_03_difficult']
# vector_w = T * vector_b
#read csv
train_image_list = []
train_pose_list = []
for seq in train_seq:
path_csv = os.path.join(root, seq, 'data.csv')
df = pd.DataFrame(pd.read_csv(path_csv))
image_list = df['filename'].values.tolist()
image_list = [
os.path.join(root, seq, 'image', name) for name in image_list
]
pose_list = []
for row in df.iterrows():
index, data = row
pose_list.append(data.tolist()[1:-1])
pose_list = [util.vector_to_transform(item) for item in pose_list]
train_image_list.append(image_list)
train_pose_list.append(pose_list)
# train_dataset = base_dataset(image_list[0:int(len(image_list)*split)], pose_list[0:int(len(image_list)*split)], stride, transform)
train_dataset = base_dataset(train_image_list, train_pose_list, stride,
transform)
test_image_list = []
test_pose_list = []
for seq in test_seq:
path_csv = os.path.join(root, seq, 'data.csv')
df = pd.DataFrame(pd.read_csv(path_csv))
image_list = df['filename'].values.tolist()
image_list = [
os.path.join(root, seq, 'image', name) for name in image_list
]
pose_list = []
for row in df.iterrows():
index, data = row
pose_list.append(data.tolist()[1:-1])
pose_list = [util.vector_to_transform(item) for item in pose_list]
test_image_list.append(image_list)
test_pose_list.append(pose_list)
# test_dataset = base_dataset(image_list[int(len(image_list)*split):], pose_list[int(len(image_list)*split):], stride, transform)
test_dataset = base_dataset(test_image_list, test_pose_list, stride,
transform)
return train_dataset, test_dataset
def load_csv(self, name="labels.csv"):
self.data = []
for d in self.dir:
data = pd.read_csv(os.path.join(d, name), skiprows=None)
data = data.values
data = data.tolist()
data = [[d + "/ori/" + _[0].replace("jpg", "png"), _[1]] for _ in data]
self.data += data
def crop_center(pil_img, crop_width, crop_height):
img_width, img_height = pil_img.size
pil_img.crop(
((img_width - crop_width) // 2, (img_height - crop_height) // 2,
(img_width + crop_width) // 2, (img_height + crop_height) // 2))
pil_img = pil_img.resize((224, 224))
data = np.asarray(pil_img)
data = data.tolist()
return data
def split_train_test(datasets, targets, test_ratio):
np.random.seed(random.randrange(50))
shuffled_indices = np.random.permutation(len(datasets))
test_set_size = int(len(datasets) * test_ratio)
test_indices = shuffled_indices[:test_set_size]
train_indices = shuffled_indices[test_set_size:]
return datasets.iloc[train_indices].values.tolist(), \
datasets.iloc[test_indices].values.tolist(), \
targets.iloc[train_indices].values.tolist(), \
targets.iloc[test_indices].values.tolist()
train_data, test_data, train_target, test_target = split_train_test(
pd.DataFrame(data.tolist()), pd.DataFrame(target.tolist()), 0.3)
train_data, test_data, train_target, test_target = torch.Tensor(
train_data), torch.Tensor(test_data), torch.Tensor(
train_target), torch.Tensor(test_target)
def train_model(epoch, net, train_data, train_target):
# for i in range(0, train_data.shape[0], batch_size):
# if i + batch_size > train_data.shape[0]:
# end = train_data.shape[0]
# else:
# end = i + batch_size
# batch_data = train_data[i:end]
# batch_data = batch_data.view(-1, 300)
# batch_data = batch_data.to(device)
#
def plot_dist(data,
logger,
name="undefined",
ran=[],
save_path="./",
bins=100,
kde=False,
hist=True,
notes="",
redo=False,
rug_max=False,
stat=True,
cumulative=False):
if not os.path.exists(save_dir):
os.makedirs(save_dir)
if not isinstance(data, np.ndarray):
data = np.array(data)
data = data.flatten()
data = np.sort(data)
stat_data = "mean=%.4f var=%.4f, min=%.4f, max=%.4f mid=%.4f" % (np.mean(
data), np.var(data), np.min(data), np.max(data), np.median(data))
num_max = 100000
if data.shape[0] > num_max:
print("data is too large. sample %d elements.")
data = np.array(random.sample(data.tolist(), num_max))
logger.write(stat_data)
img_dir = {}
y_interv = 0.05
y_line = 0.95
#hist_plot
if hist:
sig = True
sig_check = False
while sig:
try:
title = name + " distribution"
if (cumulative == True):
title += "(cumu)"
hist_dir = save_dir + title + ".jpg"
img_dir["hist"] = hist_dir
if (redo == False and os.path.exists(hist_dir)):
print("image already exists.")
else:
plt.figure()
if (ran != []):
plt.xlim(ran[0], ran[1])
else:
set_lim(data, None, plt)
#sns.distplot(data, bins=bins, color='b',kde=False)
#method="sns"
plt.hist(data,
bins=bins,
color="b",
density=True,
cumulative=cumulative)
method = "hist"
plt.title(title, fontsize=large_fontsize)
if (stat == True):
plt.annotate(stat_data,
xy=(0.02, y_line),
xycoords='axes fraction')
y_line -= y_interv
print_notes(notes, y_line, y_interv)
plt.savefig(hist_dir)
plt.close()
sig = False
except Exception:
if (sig_check == True):
raise Exception("exception in plot dist.")
data, note = check_array_1d(data, logger)
if (isinstance(notes, str)):
notes = [notes, note]
elif (isinstance(notes, list)):
notes = notes + [note]
sig_check = True
try: #kde_plot
y_line = 0.95
if kde:
title = name + " dist(kde)"
if (cumulative == True):
title += "(cumu)"
kde_dir = save_dir + title + ".jpg"
img_dir["kde"] = kde_dir
if (redo == False and os.path.exists(kde_dir)):
print("image already exists.")
else:
plt.figure()
if (ran != []):
plt.xlim(ran[0], ran[1])
else:
set_lim(data, None, plt)
sns.kdeplot(data, shade=True, color='b', cumulative=cumulative)
if rug_max:
sns.rugplot(data[(data.shape[0] -
data.shape[0] // 500):-1],
height=0.2,
color='r')
plt.title(title, fontsize=large_fontsize)
if stat:
plt.annotate(stat_data,
xy=(0.02, y_line),
xycoords='axes fraction')
y_line -= 0.05
print_notes(notes, y_line, y_interv)
plt.savefig(kde_dir)
plt.close()
except Exception:
print("exception in kde plot %s" % (kde_dir))
return img_dir
def _deserialize_array(self, x):
data = array.array(self.dtype)
data.frombytes(x)
return data.tolist()
teacher = torch.load(f)
optimizer = optim.Adam(student.parameters(), lr=lr)
train_set = datasets.MNIST('../data',
train=True,
download=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))
]))
transfer_data, transfer_labels = [], []
for data, label in train_set:
if label != 3:
transfer_data.append(data.tolist())
transfer_labels.append(label)
transfer_data, transfer_labels = torch.Tensor(transfer_data), torch.Tensor(
transfer_labels)
kwargs = {'num_workers': 1, 'pin_memory': True}
transfer_loader = torch.utils.data.DataLoader(torch.utils.data.TensorDataset(
transfer_data, transfer_labels),
batch_size=batch_size,
shuffle=True,
**kwargs)
test_loader = torch.utils.data.DataLoader(datasets.MNIST(
'../data',
train=False,
#%%
counter = 0
for key_ in info2index:
print(key_)
print(info2index[key_])
counter += 1
if counter == 5:
break
#%%
event_embeddings = pd.read_csv(repo_dir + "/word_embeddings/u_epoch_500.csv",
header=None)
embedding_lst = []
for row in event_embeddings.iterrows():
index, data = row
temp = data.tolist()
actual_data = [float(x) for x in temp[0].split()]
embedding_lst.append(actual_data)
#%%
number_to_month = {
"01": "Jan",
"02": "Feb",
"03": "Mar",
"04": "Apr",
"05": "May",
"06": "Jun",
"07": "Jul",
"08": "Aug",
"09": "Sep",
"10": "Oct",