def main():
data_size = 525000
x_mean = 22.5
x_sigma = 0.5
Datas = np.random.normal(loc=x_mean, scale=x_sigma, size=(data_size))
Datas = list(map(lambda i: int(round(i)), Datas))
Min_V = min(Datas)
Max_V = max(Datas)
Means = list(x_mean * np.ones(data_size))
Scales = list(x_sigma * np.ones(data_size))
T1 = time.time()
AE.encode(Datas, Means, Scales, "./compressed.bin")
TE1 = time.time() - T1
AE.init_decoder("./compressed.bin", Min_V, Max_V)
T2 = time.time()
Recons = []
for i in range(data_size):
Recons.append(AE.decode(Means[i], Scales[i]))
TE2 = time.time() - T2
error_number = 0
for i in range(data_size):
if Datas[i] != Recons[i]:
error_number += 1
print("Error Number: ", error_number)
print("Encoding Time: ", TE1)
print("Decoding Time: ", TE2)
print("Total Time: ", TE1 + TE2)
def test():
target = AE.AECreateDesc('sign', 'KAHL')
ae = AE.AECreateAppleEvent('aevt', 'oapp', target, -1, 0)
print unpackevent(ae)
raw_input(":")
ae = AE.AECreateAppleEvent('core', 'getd', target, -1, 0)
obj = Character(2, Word(1, Document(1)))
print obj
print repr(obj)
packevent(ae, {'----': obj})
params, attrs = unpackevent(ae)
print params['----']
raw_input(":")
def main():
data_size = 525000
x_mean = 25
x_sigma = 0.5
Datas = np.random.normal(loc=x_mean, scale=x_sigma, size=(data_size))
Datas = list(map(lambda i: int(round(i)), Datas))
Min_V = min(Datas)
Max_V = max(Datas)
Means = x_mean * np.ones(data_size)
Scales = x_sigma * np.ones(data_size)
range_size = Max_V - Min_V + 1
factor = (1 << 16) - range_size
cdf = scipy.stats.norm(x_mean - Min_V,
x_sigma).cdf(np.arange(range_size + 1) - 0.5)
for i in range(range_size + 1):
cdf[i] = cdf[i] * factor + i + 1
cdf = cdf.astype(np.int)
print(cdf)
Cdf = np.tile(cdf, [data_size, 1])
T1 = time.time()
Cdf_0 = list(map(lambda x, y: int(y[x - Min_V]), Datas, Cdf))
Cdf_1 = list(map(lambda x, y: int(y[x - Min_V]), Datas, Cdf[:, 1:]))
AE.encode_cdf(Cdf_0, Cdf_1, "./compressed.bin")
TE1 = time.time() - T1
AE.init_decoder("./compressed.bin", Min_V, Max_V)
T2 = time.time()
Recons = []
for i in range(data_size):
Recons.append(AE.decode_cdf(cdf.tolist()))
TE2 = time.time() - T2
error_number = 0
for i in range(data_size):
if Datas[i] != Recons[i]:
error_number += 1
print("Error Number: ", error_number)
print("Encoding Time: ", TE1)
print("Decoding Time: ", TE2)
print("Total Time: ", TE1 + TE2)
def calculate_faed_given_paths(paths, batch_size, cuda, dims):
"""Calculates the FID of two paths"""
cuda = True
for p in paths:
p=p.split('\\')[0]
if not os.path.exists(p):
raise RuntimeError('Invalid path: %s' % p)
# block_idx = InceptionV3.BLOCK_INDEX_BY_DIM[dims]
# model = InceptionV3([block_idx])
model_path="./AE_dict_model.pt"
input_dim=4096*3
model=AE.load_encoder(model_path,input_dim)
#Reference stat
mu_saved_data_label=os.path.join("./saved_ref","mu_"+(paths[0].split("\\")[1]))
sig_saved_data_label=os.path.join("./saved_ref","sig_"+(paths[0].split("\\")[1]))
if (os.path.isfile(mu_saved_data_label) and os.path.isfile(sig_saved_data_label)):
m1=np.load(mu_saved_data_label)['x']
s1=np.load(sig_saved_data_label)['x']
else:
m1, s1 = _compute_statistics_of_path(paths[0], model, batch_size,
dims, cuda)
np.savez(mu_saved_data_label,**{'x':m1})
np.savez(sig_saved_data_label,**{'x':s1})
m2, s2 = _compute_statistics_of_path(paths[1], model, batch_size,
dims, cuda)
fid_value = calculate_frechet_distance(m1, s1, m2, s2)
return fid_value
def passtothink(list):
target = AE.AECreateDesc(typeApplSignature, 'KAHL')
event = AE.AECreateAppleEvent(kCoreEventClass,
kAEOpenDocuments,
target,
kAutoGenerateReturnID,
kAnyTransactionID)
aetools.packevent(event, {keyDirectObject: list})
reply = event.AESend(kAENoReply | kAEAlwaysInteract | kAECanSwitchLayer,
kAENormalPriority,
kAEDefaultTimeout)
#print "Reply:", aetools.unpackevent(reply)
return
event = AE.AECreateAppleEvent(kCoreEventClass,
kAEOpenApplication,
target,
kAutoGenerateReturnID,
kAnyTransactionID)
reply = event.AESend(kAENoReply | kAEAlwaysInteract | kAECanSwitchLayer,
kAENormalPriority,
kAEDefaultTimeout)
def __init__(self):
self.active = 0
for suite in self.suites:
AE.AEInstallEventHandler(suite, typeWildCard, self.aehandler)
self.active = 1
self.appleid = 1
Menu.ClearMenuBar()
self.applemenu = applemenu = Menu.NewMenu(self.appleid, "\024")
applemenu.AppendMenu("All about echo...;(-")
applemenu.AddResMenu('DRVR')
applemenu.InsertMenu(0)
Menu.DrawMenuBar()
def main(): global quit quit = 0 while not quit: ok, e = Evt.WaitNextEvent(-1, 60) if ok: print 'Event:', e if e[0] == 23: # kHighLevelEvent AE.AEProcessAppleEvent(e) elif e[0] == keyDown and chr(e[1]&0xff) == '.' and e[4]&cmdKey: raise KeyboardInterrupt, "Command-Period" else: MacOS.HandleEvent(e)
def lowlevelhandler(self, event): what, message, when, where, modifiers = event h, v = where if what == kHighLevelEvent: msg = "High Level Event: %s %s" % \ (`code(message)`, `code(h | (v<<16))`) try: AE.AEProcessAppleEvent(event) except AE.Error, err: EasyDialogs.Message(msg + "\015AEProcessAppleEvent error: %s" % str(err)) traceback.print_exc() else: EasyDialogs.Message(msg + "\015OK!")
def pack(x, forcetype = None):
if forcetype:
if type(x) is StringType:
return AE.AECreateDesc(forcetype, x)
else:
return pack(x).AECoerceDesc(forcetype)
if x == None:
return AE.AECreateDesc('null', '')
t = type(x)
if t == AEDescType:
return x
if t == FSSType:
vol, dir, filename = x.as_tuple()
fnlen = len(filename)
header = struct.pack('hlb', vol, dir, fnlen)
padding = '\0'*(63-fnlen)
return AE.AECreateDesc('fss ', header + filename + padding)
if t == IntType:
return AE.AECreateDesc('long', struct.pack('l', x))
if t == FloatType:
# XXX Weird thing -- Think C's "double" is 10 bytes, but
# struct.pack('d') return 12 bytes (and struct.unpack requires
# them, too). The first 2 bytes seem to be repeated...
# Probably an alignment problem
return AE.AECreateDesc('exte', struct.pack('d', x)[2:])
if t == StringType:
return AE.AECreateDesc('TEXT', x)
if t == ListType:
list = AE.AECreateList('', 0)
for item in x:
list.AEPutDesc(0, pack(item))
return list
if t == DictionaryType:
record = AE.AECreateList('', 1)
for key, value in x.items():
record.AEPutKeyDesc(key, pack(value))
return record
if t == InstanceType and hasattr(x, '__aepack__'):
return x.__aepack__()
return AE.AECreateDesc('TEXT', repr(x)) # Copout
migration_threshold = input('migration_threshold = ')
op = input('type 0 to use PCA, 1 to use AE : ')
f = Feature(node_threshold=180, migration_threshold=migration_threshold)
X, Y = f.getYearFeatures(2015)
X0 = []
X1 = []
for i in range(len(X)):
X0.append(X[i][0])
X1.append(X[i][1])
if op == 0:
x0 = DR_PCA(X0, dim)
x1 = DR_PCA(X1, dim)
else:
X0 = np.array(X0)
X1 = np.array(X1)
x, w, b = AE.dim_reduce(X0, dim, 2, 20, 0.01)
x0 = AE.forward2hidden(X0, w, b, 2)
x, w, b = AE.dim_reduce(X1, dim, 2, 20, 0.01)
x1 = AE.forward2hidden(X1, w, b, 2)
x = []
for i in range(len(X)):
x.append(np.concatenate((x0[i], x1[i], X[i][2])))
print('Dimension reduction done.')
x = np.array(x)
dim = 2 * dim + 3
num = int(sqrt(len(x)))
O = [sum([Y[i * num + j] for j in range(num)]) for i in range(num)]
I = [sum([Y[i * num + j] for i in range(num)]) for j in range(num)]
print 'core event!'
parameters, attributes = aetools.unpackevent(req)
print "event class =", attributes['evcl']
print "event id =", attributes['evid']
print 'parameters:', parameters
# echo the arguments, to see how Script Editor formats them
aetools.packevent(rep, parameters)
def wildhandler(req, rep):
print 'wildcard event!'
parameters, attributes = aetools.unpackevent(req)
print "event class =", attributes['evcl']
print "event id =", attributes['evid']
print 'parameters:', parameters
AE.AEInstallEventHandler(typeAppleEvent, kAEOpenApplication, aehandler)
AE.AEInstallEventHandler(typeAppleEvent, kAEOpenDocuments, aehandler)
AE.AEInstallEventHandler(typeAppleEvent, kAEPrintDocuments, aehandler)
AE.AEInstallEventHandler(typeAppleEvent, kAEQuitApplication, quithandler)
AE.AEInstallEventHandler(typeAppleEvent, typeWildCard, unihandler)
AE.AEInstallEventHandler('core', typeWildCard, corehandler)
#AE.AEInstallEventHandler(typeWildCard, typeWildCard, wildhandler)
def main():
global quit
quit = 0
while not quit:
ok, e = Evt.WaitNextEvent(-1, 60)
if ok:
print 'Event:', e
import sys
sys.path.insert(0, '../lib/pylib/')
from preprocess_10x import getNormAnnData
import AE
if (sys.argv[2] == 'log'):
adata, size_factor = getNormAnnData(sys.argv[1], sys.argv[2])
AE.train_zinb_model(adata, size_factor)
AE.prediction_zinb_middle(adata, size_factor, sys.argv[3])
else:
adata = getNormAnnData(sys.argv[1], sys.argv[2])
AE.train_zip_model(adata)
AE.prediction_zip_middle(adata, sys.argv[3])
def decode(bin_dir, rec_dir, model_dir, block_width, block_height):
############### retreive head info ###############
T = time.time()
file_object = open(bin_dir, 'rb')
head_len = struct.calcsize('2HB')
bits = file_object.read(head_len)
[H, W, model_index] = struct.unpack('2HB', bits)
# print("File Info:",Head)
# Split Main & Hyper bins
C = 3
out_img = np.zeros([H, W, C])
H_offset = 0
W_offset = 0
Block_Num_in_Width = int(np.ceil(W / block_width))
Block_Num_in_Height = int(np.ceil(H / block_height))
c_main = 192
c_hyper = 128
M, N2 = 192, 128
if (model_index == 6) or (model_index == 7) or (model_index
== 14) or (model_index
== 15):
M, N2 = 256, 192
image_comp = model.Image_coding(3, M, N2, M, M // 2)
context = Weighted_Gaussian(M)
######################### Load Model #########################
image_comp.load_state_dict(
torch.load(os.path.join(model_dir, models[model_index] + r'.pkl'),
map_location='cpu'))
context.load_state_dict(
torch.load(os.path.join(model_dir, models[model_index] + r'p.pkl'),
map_location='cpu'))
if GPU:
image_comp.cuda()
context.cuda()
for i in range(Block_Num_in_Height):
for j in range(Block_Num_in_Width):
Block_head_len = struct.calcsize('2H4h2I')
bits = file_object.read(Block_head_len)
[
block_H, block_W, Min_Main, Max_Main, Min_V_HYPER, Max_V_HYPER,
FileSizeMain, FileSizeHyper
] = struct.unpack('2H4h2I', bits)
precise, tile = 16, 64.
block_H_PAD = int(tile * np.ceil(block_H / tile))
block_W_PAD = int(tile * np.ceil(block_W / tile))
with open("main.bin", 'wb') as f:
bits = file_object.read(FileSizeMain)
f.write(bits)
with open("hyper.bin", 'wb') as f:
bits = file_object.read(FileSizeHyper)
f.write(bits)
############### Hyper Decoder ###############
# [Min_V - 0.5 , Max_V + 0.5]
sample = np.arange(Min_V_HYPER, Max_V_HYPER + 1 + 1)
sample = np.tile(sample, [c_hyper, 1, 1])
lower = torch.sigmoid(
image_comp.factorized_entropy_func._logits_cumulative(
torch.FloatTensor(sample) - 0.5, stop_gradient=False))
cdf_h = lower.data.cpu().numpy() * (
(1 << precise) -
(Max_V_HYPER - Min_V_HYPER + 1)) # [N1, 1, Max - Min]
cdf_h = cdf_h.astype(np.int) + sample.astype(np.int) - Min_V_HYPER
T2 = time.time()
AE.init_decoder("hyper.bin", Min_V_HYPER, Max_V_HYPER)
Recons = []
for i in range(c_hyper):
for j in range(int(block_H_PAD * block_W_PAD / 64 / 64)):
# print(cdf_h[i,0,:])
Recons.append(AE.decode_cdf(cdf_h[i, 0, :].tolist()))
# reshape Recons to y_hyper_q [1, c_hyper, H_PAD/64, W_PAD/64]
y_hyper_q = torch.reshape(
torch.Tensor(Recons),
[1, c_hyper,
int(block_H_PAD / 64),
int(block_W_PAD / 64)])
############### Main Decoder ###############
hyper_dec = image_comp.p(image_comp.hyper_dec(y_hyper_q))
h, w = int(block_H_PAD / 16), int(block_W_PAD / 16)
sample = np.arange(Min_Main,
Max_Main + 1 + 1) # [Min_V - 0.5 , Max_V + 0.5]
sample = torch.FloatTensor(sample)
p3d = (5, 5, 5, 5, 5, 5)
y_main_q = torch.zeros(1, 1, c_main + 10, h + 10,
w + 10) # 8000x4000 -> 500*250
AE.init_decoder("main.bin", Min_Main, Max_Main)
hyper = torch.unsqueeze(context.conv3(hyper_dec), dim=1)
#
context.conv1.weight.data *= context.conv1.mask
for i in range(c_main):
T = time.time()
for j in range(int(block_H_PAD / 16)):
for k in range(int(block_W_PAD / 16)):
x1 = F.conv3d(y_main_q[:, :, i:i + 12, j:j + 12,
k:k + 12],
weight=context.conv1.weight,
bias=context.conv1.bias) # [1,24,1,1,1]
params_prob = context.conv2(
torch.cat(
(x1, hyper[:, :, i:i + 2, j:j + 2, k:k + 2]),
dim=1))
# 3 gaussian
prob0, mean0, scale0, prob1, mean1, scale1, prob2, mean2, scale2 = params_prob[
0, :, 0, 0, 0]
# keep the weight summation of prob == 1
probs = torch.stack([prob0, prob1, prob2], dim=-1)
probs = F.softmax(probs, dim=-1)
# process the scale value to positive non-zero
scale0 = torch.abs(scale0)
scale1 = torch.abs(scale1)
scale2 = torch.abs(scale2)
scale0[scale0 < 1e-6] = 1e-6
scale1[scale1 < 1e-6] = 1e-6
scale2[scale2 < 1e-6] = 1e-6
# 3 gaussian distributions
m0 = torch.distributions.normal.Normal(
mean0.view(1, 1).repeat(1,
Max_Main - Min_Main + 2),
scale0.view(1, 1).repeat(1,
Max_Main - Min_Main + 2))
m1 = torch.distributions.normal.Normal(
mean1.view(1, 1).repeat(1,
Max_Main - Min_Main + 2),
scale1.view(1, 1).repeat(1,
Max_Main - Min_Main + 2))
m2 = torch.distributions.normal.Normal(
mean2.view(1, 1).repeat(1,
Max_Main - Min_Main + 2),
scale2.view(1, 1).repeat(1,
Max_Main - Min_Main + 2))
lower0 = m0.cdf(sample - 0.5)
lower1 = m1.cdf(sample - 0.5)
lower2 = m2.cdf(sample - 0.5) # [1,c,h,w,Max-Min+2]
lower = probs[0:1] * lower0 + probs[
1:2] * lower1 + probs[2:3] * lower2
cdf_m = lower.data.cpu().numpy() * (
(1 << precise) - (Max_Main - Min_Main + 1)
) # [1, c, h, w ,Max-Min+1]
cdf_m = cdf_m.astype(np.int) + \
sample.numpy().astype(np.int) - Min_Main
pixs = AE.decode_cdf(cdf_m[0, :].tolist())
y_main_q[0, 0, i + 5, j + 5, k + 5] = pixs
print("Decoding Channel (%d/192), Time (s): %0.4f" %
(i, time.time() - T))
del hyper, hyper_dec
y_main_q = y_main_q[0, :, 5:-5, 5:-5, 5:-5]
rec = image_comp.decoder(y_main_q)
output_ = torch.clamp(rec, min=0., max=1.0)
out = output_.data[0].cpu().numpy()
out = out.transpose(1, 2, 0)
out_img[H_offset:H_offset + block_H, W_offset:W_offset +
block_W, :] = out[:block_H, :block_W, :]
W_offset += block_W
if W_offset >= W:
W_offset = 0
H_offset += block_H
out_img = np.round(out_img * 255.0)
out_img = out_img.astype('uint8')
img = Image.fromarray(out_img[:H, :W, :])
img.save(rec_dir)
character 1 of document "foobar"
"""
import struct
import string
from string import strip
from types import *
import AE
import MacOS
import macfs
import StringIO
AEDescType = type(AE.AECreateDesc('TEXT', ''))
FSSType = type(macfs.FSSpec(':'))
def pack(x, forcetype = None):
if forcetype:
if type(x) is StringType:
return AE.AECreateDesc(forcetype, x)
else:
return pack(x).AECoerceDesc(forcetype)
if x == None:
return AE.AECreateDesc('null', '')
t = type(x)
if t == AEDescType:
return x
local_path = os.path.dirname(os.path.abspath(__file__))
# Get arguments
parser=argparse.ArgumentParser()
parser.add_argument('--file', '-f', help='Configuration file', default="config/config_AE.json")
parser.add_argument('--config', '-c', help='Configuration type', default="DEFAULT")
loss = []
nbr_iterations = 10
list_size = [2, 3, 5, 7, 10, 15]
for size_latent in list_size:
for k in range(nbr_iterations):
autoencoder = AE.ModelAutoencoder(parser, local_path)
list_metric = autoencoder.get_list_metric()
metric = list_metric[0]
# size_latent = autoencoder.get_config()['latent_dim']
type_AE = autoencoder.get_config()['type_AE']
path = local_path + "/save/" + type_AE + "/" + metric + '/' + str(size_latent) + '/'
if not(os.path.isdir(path)):
os.mkdir(path)
if not(os.path.isdir(path + 'loss/')):
os.mkdir(path + 'loss/')
if os.path.exists(path + "autoencoder_" + str(size_latent) + '_' + str(k) + ".pkl"):
continue
def encode(im_dir, out_dir, model_dir, model_index, block_width, block_height):
file_object = open(out_dir, 'wb')
M, N2 = 192, 128
if (model_index == 6) or (model_index == 7) or (model_index
== 14) or (model_index
== 15):
M, N2 = 256, 192
image_comp = model.Image_coding(3, M, N2, M, M // 2)
context = Weighted_Gaussian(M)
######################### Load Model #########################
image_comp.load_state_dict(
torch.load(os.path.join(model_dir, models[model_index] + r'.pkl'),
map_location='cpu'))
context.load_state_dict(
torch.load(os.path.join(model_dir, models[model_index] + r'p.pkl'),
map_location='cpu'))
if GPU:
image_comp.cuda()
context.cuda()
######################### Read Image #########################
img = Image.open(im_dir)
img = np.array(img) / 255.0
H, W, _ = img.shape
num_pixels = H * W
C = 3
Head = struct.pack('2HB', H, W, model_index)
file_object.write(Head)
######################### spliting Image #########################
Block_Num_in_Width = int(np.ceil(W / block_width))
Block_Num_in_Height = int(np.ceil(H / block_height))
img_block_list = []
for i in range(Block_Num_in_Height):
for j in range(Block_Num_in_Width):
img_block_list.append(
img[i * block_height:np.minimum((i + 1) * block_height, H),
j * block_width:np.minimum((j + 1) * block_width, W), ...])
######################### Padding Image #########################
Block_Idx = 0
for img in img_block_list:
block_H = img.shape[0]
block_W = img.shape[1]
tile = 64.
block_H_PAD = int(tile * np.ceil(block_H / tile))
block_W_PAD = int(tile * np.ceil(block_W / tile))
im = np.zeros([block_H_PAD, block_W_PAD, 3], dtype='float32')
im[:block_H, :block_W, :] = img[:, :, :3]
im = torch.FloatTensor(im)
im = im.permute(2, 0, 1).contiguous()
im = im.view(1, C, block_H_PAD, block_W_PAD)
if GPU:
im = im.cuda()
print('====> Encoding Image:', im_dir, "%dx%d" % (block_H, block_W),
'to', out_dir, " Block Idx: %d" % (Block_Idx))
Block_Idx += 1
with torch.no_grad():
y_main, y_hyper = image_comp.encoder(im)
y_main_q = torch.round(y_main)
y_main_q = torch.Tensor(y_main_q.numpy().astype(np.int))
# y_hyper_q = torch.round(y_hyper)
y_hyper_q, xp2 = image_comp.factorized_entropy_func(y_hyper, 2)
y_hyper_q = torch.Tensor(y_hyper_q.numpy().astype(np.int))
hyper_dec = image_comp.p(image_comp.hyper_dec(y_hyper_q))
xp3, params_prob = context(y_main_q, hyper_dec)
# Main Arith Encode
Datas = torch.reshape(y_main_q,
[-1]).cpu().numpy().astype(np.int).tolist()
Max_Main = max(Datas)
Min_Main = min(Datas)
sample = np.arange(Min_Main,
Max_Main + 1 + 1) # [Min_V - 0.5 , Max_V + 0.5]
_, c, h, w = y_main_q.shape
print("Main Channel:", c)
sample = torch.FloatTensor(np.tile(sample, [1, c, h, w, 1]))
# 3 gaussian
prob0, mean0, scale0, prob1, mean1, scale1, prob2, mean2, scale2 = [
torch.chunk(params_prob, 9, dim=1)[i].squeeze(1) for i in range(9)
]
del params_prob
# keep the weight summation of prob == 1
probs = torch.stack([prob0, prob1, prob2], dim=-1)
del prob0, prob1, prob2
probs = F.softmax(probs, dim=-1)
# process the scale value to positive non-zero
scale0 = torch.abs(scale0)
scale1 = torch.abs(scale1)
scale2 = torch.abs(scale2)
scale0[scale0 < 1e-6] = 1e-6
scale1[scale1 < 1e-6] = 1e-6
scale2[scale2 < 1e-6] = 1e-6
m0 = torch.distributions.normal.Normal(mean0, scale0)
m1 = torch.distributions.normal.Normal(mean1, scale1)
m2 = torch.distributions.normal.Normal(mean2, scale2)
lower = torch.zeros(1, c, h, w, Max_Main - Min_Main + 2)
for i in range(sample.shape[4]):
# print("CDF:", i)
lower0 = m0.cdf(sample[:, :, :, :, i] - 0.5)
lower1 = m1.cdf(sample[:, :, :, :, i] - 0.5)
lower2 = m2.cdf(sample[:, :, :, :, i] - 0.5)
lower[:, :, :, :, i] = probs[:, :, :, :, 0]*lower0 + \
probs[:, :, :, :, 1]*lower1+probs[:, :, :, :, 2]*lower2
del probs, lower0, lower1, lower2
precise = 16
cdf_m = lower.data.cpu().numpy() * (
(1 << precise) -
(Max_Main - Min_Main + 1)) # [1, c, h, w ,Max-Min+1]
cdf_m = cdf_m.astype(np.int32) + sample.numpy().astype(
np.int32) - Min_Main
cdf_main = np.reshape(cdf_m, [len(Datas), -1])
# Cdf[Datas - Min_V]
Cdf_lower = list(
map(lambda x, y: int(y[x - Min_Main]), Datas, cdf_main))
# Cdf[Datas + 1 - Min_V]
Cdf_upper = list(
map(lambda x, y: int(y[x - Min_Main]), Datas, cdf_main[:, 1:]))
AE.encode_cdf(Cdf_lower, Cdf_upper, "main.bin")
FileSizeMain = os.path.getsize("main.bin")
print("main.bin: %d bytes" % (FileSizeMain))
# Hyper Arith Encode
Min_V_HYPER = torch.min(y_hyper_q).cpu().numpy().astype(
np.int).tolist()
Max_V_HYPER = torch.max(y_hyper_q).cpu().numpy().astype(
np.int).tolist()
_, c, h, w = y_hyper_q.shape
# print("Hyper Channel:", c)
Datas_hyper = torch.reshape(y_hyper_q, [c, -1]).cpu().numpy().astype(
np.int).tolist()
# [Min_V - 0.5 , Max_V + 0.5]
sample = np.arange(Min_V_HYPER, Max_V_HYPER + 1 + 1)
sample = np.tile(sample, [c, 1, 1])
lower = torch.sigmoid(
image_comp.factorized_entropy_func._logits_cumulative(
torch.FloatTensor(sample) - 0.5, stop_gradient=False))
cdf_h = lower.data.cpu().numpy() * (
(1 << precise) -
(Max_V_HYPER - Min_V_HYPER + 1)) # [N1, 1, Max-Min+1]
cdf_h = cdf_h.astype(np.int) + sample.astype(np.int) - Min_V_HYPER
cdf_hyper = np.reshape(np.tile(cdf_h, [len(Datas_hyper[0]), 1, 1, 1]),
[len(Datas_hyper[0]), c, -1])
# Datas_hyper [256, N], cdf_hyper [256,1,X]
Cdf_0, Cdf_1 = [], []
for i in range(c):
Cdf_0.extend(
list(
map(lambda x, y: int(y[x - Min_V_HYPER]), Datas_hyper[i],
cdf_hyper[:, i, :]))) # Cdf[Datas - Min_V]
Cdf_1.extend(
list(
map(lambda x, y: int(y[x - Min_V_HYPER]), Datas_hyper[i],
cdf_hyper[:, i, 1:]))) # Cdf[Datas + 1 - Min_V]
AE.encode_cdf(Cdf_0, Cdf_1, "hyper.bin")
FileSizeHyper = os.path.getsize("hyper.bin")
print("hyper.bin: %d bytes" % (FileSizeHyper))
Head_block = struct.pack('2H4h2I', block_H, block_W, Min_Main,
Max_Main, Min_V_HYPER, Max_V_HYPER,
FileSizeMain, FileSizeHyper)
file_object.write(Head_block)
# cat Head_Infor and 2 files together
# Head = [FileSizeMain,FileSizeHyper,H,W,Min_Main,Max_Main,Min_V_HYPER,Max_V_HYPER,model_index]
# print("Head Info:",Head)
with open("main.bin", 'rb') as f:
bits = f.read()
file_object.write(bits)
with open("hyper.bin", 'rb') as f:
bits = f.read()
file_object.write(bits)
import AE
from sklearn.preprocessing import StandardScaler
import numpy as np
import pandas as pd
if __name__ == "__main__":
all_data_encoded = np.load("sample/data.npy")
model = AE.MyModel()
model.compile(optimizer="adam", loss="mean_squared_error")
num_epochs = 10000
batch_size = 64
history = model.fit(x=all_data_encoded,
y=all_data_encoded,
epochs=num_epochs,
batch_size=batch_size,
shuffle=True,
verbose=1)
np.save("sample/latent.npy", model.encode(all_data_encoded).numpy())
def close(self): if self.active: self.active = 0 for suite in self.suites: AE.AERemoveEventHandler(suite, typeWildCard)
pca = PCA(n_components = DIM)
train_X_reduce = np.concatenate((
pca.fit_transform(np.array([ x[0]+x[1] for x in train_X ])),
np.array([ x[2] for x in train_X ])
), axis=1)
# Applies same model on test data.
test_X_reduce = np.concatenate((
pca.transform(np.array([ x[0]+x[1] for x in test_X ])),
np.array([ x[2] for x in test_X ])
), axis=1)
print ("Dimension reduction done with PCA")
else:
feature.standardizeX(train_X)
feature.standardizeX(test_X)
train_X_reduce, ae_w, ae_b = AE.dim_reduce(
np.array([ x[0]+x[1] for x in train_X ]), DIM, BATCH, 50, 0.01
)
train_X_reduce = np.concatenate((
train_X_reduce, np.array([ x[2] for x in train_X ])
), axis=1)
# Applies same model on test data
test_X_reduce = np.concatenate((
AE.forward2hidden(
np.array([ x[0]+x[1] for x in test_X ]), ae_w, ae_b, DIM
), np.array([ x[2] for x in test_X ])
), axis=1)
print ("Dimension reduction done with AE")
clf = svm.SVR(verbose=True)
clf.fit(train_X_reduce, train_Y)
losses = []
for i_batch, batch in enumerate(iterator):
x_batch = batch.squeeze(0)
x_batch = x_batch.to(DEVICE).float()
out, z_batch, mean, log_var = model(x_batch)
loss = compute_loss(x_batch, z_batch, mean, log_var, out)
losses.append(loss.item())
return np.mean(losses)
if __name__ == '__main__':
import time
model = AE().to(DEVICE)
model.cuda()
opt = torch.optim.Adam(model.parameters())
training_dataset = DoomDataset(TRAIN_DIR, 3)
training_iterator = DataLoader(training_dataset,
batch_size=1,
shuffle=True)
for epoch in range(EPOCHS):
print(epoch)
train_vae(model, training_iterator, opt, time.time())
torch.save(model.state_dict(), 'vae_final.weights')
torch.save(model.state_dict(), 'vae_final.weights')
def inference_rd(im_dirs, out_dir, model_dir, model_index, block_width, block_height):
M, N2 = 192, 128
if (model_index == 6) or (model_index == 7) or (model_index == 14) or (model_index == 15):
M, N2 = 256, 192
image_comp = model.Image_coding(3, M, N2, M, M//2)
context = Weighted_Gaussian(M)
######################### Load Model #########################
image_comp.load_state_dict(torch.load(
os.path.join(model_dir, models[model_index] + r'.pkl'), map_location='cpu'))
context.load_state_dict(torch.load(
os.path.join(model_dir, models[model_index] + r'p.pkl'), map_location='cpu'))
if GPU:
image_comp.cuda()
context.cuda()
####################compress each image###################
for im_dir in im_dirs:
dec_time = 0
enc_dec_time_start = time.time()
bin_dir = os.path.join(out_dir,'enc.bin')
rec_dir = os.path.join(out_dir,'dec.png')
file_object = open(bin_dir, 'wb')
######################### Read Image #########################
img = Image.open(im_dir)
ori_img = np.array(img)
img = ori_img
H, W, _ = img.shape
num_pixels = H * W
C = 3
Head = struct.pack('2HB', H, W, model_index)
file_object.write(Head)
out_img = np.zeros([H, W, C])
H_offset = 0
W_offset = 0
######################### spliting Image #########################
Block_Num_in_Width = int(np.ceil(W / block_width))
Block_Num_in_Height = int(np.ceil(H / block_height))
img_block_list = []
for i in range(Block_Num_in_Height):
for j in range(Block_Num_in_Width):
img_block_list.append(img[i * block_height:np.minimum((i + 1) * block_height, H),j * block_width:np.minimum((j + 1) * block_width,W),...])
######################### Padding Image #########################
Block_Idx = 0
y_main_q_list = []
for img in img_block_list:
block_H = img.shape[0]
block_W = img.shape[1]
tile = 64.
block_H_PAD = int(tile * np.ceil(block_H / tile))
block_W_PAD = int(tile * np.ceil(block_W / tile))
im = np.zeros([block_H_PAD, block_W_PAD, 3], dtype='float32')
im[:block_H, :block_W, :] = img[:, :, :3]/255.0
im = torch.FloatTensor(im)
im = im.permute(2, 0, 1).contiguous()
im = im.view(1, C, block_H_PAD, block_W_PAD)
if GPU:
im = im.cuda()
print('====> Encoding Image:', im_dir, "%dx%d" % (block_H, block_W), 'to', bin_dir, " Block Idx: %d" % (Block_Idx))
Block_Idx +=1
with torch.no_grad():
y_main, y_hyper = image_comp.encoder(im)
y_main_q = torch.round(y_main)
y_main_q = torch.Tensor(y_main_q.numpy().astype(np.int))
####decoding####
dec_time_start = time.time()
rec = image_comp.decoder(y_main_q)
output_ = torch.clamp(rec, min=0., max=1.0)
out = output_.data[0].cpu().numpy()
out = out.transpose(1, 2, 0)
out_img[H_offset : H_offset + block_H, W_offset : W_offset + block_W, :] = out[:block_H, :block_W, :]
W_offset += block_W
if W_offset >= W:
W_offset = 0
H_offset += block_H
dec_time += (time.time()-dec_time_start)
# y_hyper_q = torch.round(y_hyper)
y_hyper_q, xp2 = image_comp.factorized_entropy_func(y_hyper, 2)
y_hyper_q = torch.Tensor(y_hyper_q.numpy().astype(np.int))
hyper_dec = image_comp.p(image_comp.hyper_dec(y_hyper_q))
xp3, params_prob = context(y_main_q, hyper_dec)
bpp_hyper = (torch.sum(torch.log(xp2)) / (-np.log(2) * num_pixels)).item()
bpp_main = (torch.sum(torch.log(xp3)) / (-np.log(2) * num_pixels)).item()
print('bpp_hyper_info:',bpp_hyper,'bpp_main_info:',bpp_main,'bpp_total_info:',bpp_hyper+bpp_main)
# Main Arith Encode
Datas = torch.reshape(y_main_q, [-1]).cpu().numpy().astype(np.int).tolist()
Max_Main = max(Datas)
Min_Main = min(Datas)
sample = np.arange(Min_Main, Max_Main+1+1) # [Min_V - 0.5 , Max_V + 0.5]
_, c, h, w = y_main_q.shape
print("Main Channel:", c)
sample = torch.FloatTensor(np.tile(sample, [1, c, h, w, 1]))
# 3 gaussian
prob0, mean0, scale0, prob1, mean1, scale1, prob2, mean2, scale2 = [
torch.chunk(params_prob, 9, dim=1)[i].squeeze(1) for i in range(9)]
del params_prob
# keep the weight summation of prob == 1
probs = torch.stack([prob0, prob1, prob2], dim=-1)
del prob0, prob1, prob2
probs = F.softmax(probs, dim=-1)
# process the scale value to positive non-zero
scale0 = torch.abs(scale0)
scale1 = torch.abs(scale1)
scale2 = torch.abs(scale2)
scale0[scale0 < 1e-6] = 1e-6
scale1[scale1 < 1e-6] = 1e-6
scale2[scale2 < 1e-6] = 1e-6
m0 = torch.distributions.normal.Normal(mean0, scale0)
m1 = torch.distributions.normal.Normal(mean1, scale1)
m2 = torch.distributions.normal.Normal(mean2, scale2)
lower = torch.zeros(1, c, h, w, Max_Main-Min_Main+2)
for i in range(sample.shape[4]):
# print("CDF:", i)
lower0 = m0.cdf(sample[:, :, :, :, i]-0.5)
lower1 = m1.cdf(sample[:, :, :, :, i]-0.5)
lower2 = m2.cdf(sample[:, :, :, :, i]-0.5)
lower[:, :, :, :, i] = probs[:, :, :, :, 0]*lower0 + \
probs[:, :, :, :, 1]*lower1+probs[:, :, :, :, 2]*lower2
del probs, lower0, lower1, lower2
precise = 16
cdf_m = lower.data.cpu().numpy()*((1 << precise) - (Max_Main -
Min_Main + 1)) # [1, c, h, w ,Max-Min+1]
cdf_m = cdf_m.astype(np.int32) + sample.numpy().astype(np.int32) - Min_Main
cdf_main = np.reshape(cdf_m, [len(Datas), -1])
# Cdf[Datas - Min_V]
Cdf_lower = list(map(lambda x, y: int(y[x - Min_Main]), Datas, cdf_main))
# Cdf[Datas + 1 - Min_V]
Cdf_upper = list(map(lambda x, y: int(
y[x - Min_Main]), Datas, cdf_main[:, 1:]))
AE.encode_cdf(Cdf_lower, Cdf_upper, "main.bin")
FileSizeMain = os.path.getsize("main.bin")
print("main.bin: %d bytes" % (FileSizeMain))
# Hyper Arith Encode
Min_V_HYPER = torch.min(y_hyper_q).cpu().numpy().astype(np.int).tolist()
Max_V_HYPER = torch.max(y_hyper_q).cpu().numpy().astype(np.int).tolist()
_, c, h, w = y_hyper_q.shape
# print("Hyper Channel:", c)
Datas_hyper = torch.reshape(
y_hyper_q, [c, -1]).cpu().numpy().astype(np.int).tolist()
# [Min_V - 0.5 , Max_V + 0.5]
sample = np.arange(Min_V_HYPER, Max_V_HYPER+1+1)
sample = np.tile(sample, [c, 1, 1])
lower = torch.sigmoid(image_comp.factorized_entropy_func._logits_cumulative(
torch.FloatTensor(sample) - 0.5, stop_gradient=False))
cdf_h = lower.data.cpu().numpy()*((1 << precise) - (Max_V_HYPER -
Min_V_HYPER + 1)) # [N1, 1, Max-Min+1]
cdf_h = cdf_h.astype(np.int) + sample.astype(np.int) - Min_V_HYPER
cdf_hyper = np.reshape(np.tile(cdf_h, [len(Datas_hyper[0]), 1, 1, 1]), [
len(Datas_hyper[0]), c, -1])
# Datas_hyper [256, N], cdf_hyper [256,1,X]
Cdf_0, Cdf_1 = [], []
for i in range(c):
Cdf_0.extend(list(map(lambda x, y: int(
y[x - Min_V_HYPER]), Datas_hyper[i], cdf_hyper[:, i, :]))) # Cdf[Datas - Min_V]
Cdf_1.extend(list(map(lambda x, y: int(
y[x - Min_V_HYPER]), Datas_hyper[i], cdf_hyper[:, i, 1:]))) # Cdf[Datas + 1 - Min_V]
AE.encode_cdf(Cdf_0, Cdf_1, "hyper.bin")
FileSizeHyper = os.path.getsize("hyper.bin")
print("hyper.bin: %d bytes" % (FileSizeHyper))
Head_block = struct.pack('2H4h2I', block_H, block_W, Min_Main, Max_Main, Min_V_HYPER, Max_V_HYPER, FileSizeMain, FileSizeHyper)
file_object.write(Head_block)
# cat Head_Infor and 2 files together
# Head = [FileSizeMain,FileSizeHyper,H,W,Min_Main,Max_Main,Min_V_HYPER,Max_V_HYPER,model_index]
# print("Head Info:",Head)
with open("main.bin", 'rb') as f:
bits = f.read()
file_object.write(bits)
f.close()
with open("hyper.bin", 'rb') as f:
bits = f.read()
file_object.write(bits)
f.close()
del im
file_object.close()
with open(bin_dir, "rb") as f:
bpp = len(f.read())*8./num_pixels
print('bpp_total_true:',bpp)
f.close()
out_img = np.round(out_img * 255.0)
out_img = out_img.astype('uint8')
img = Image.fromarray(out_img[:H, :W, :])
img.save(rec_dir)
[rgb_psnr, rgb_msssim, yuv_psnr,y_msssim]=evaluate(ori_img,out_img)
class_name = im_dir.split('/')[-2]
image_name = im_dir.split('/')[-1].replace('.png','')
enc_dec_time = time.time() - enc_dec_time_start
enc_time = enc_dec_time - dec_time
with open(os.path.join(out_dir,models[model_index]+'_RD.log'), "a") as f:
f.write(class_name+'/'+image_name+'\t'+str(bpp)+'\t'+str(rgb_psnr)+'\t'+str(rgb_msssim)+'\t'+str(-10*np.log10(1-rgb_msssim))+
'\t'+str(yuv_psnr)+'\t'+str(y_msssim)+'\t'+str(-10*np.log10(1-y_msssim))+
'\t'+str(enc_time)+'\t'+str(dec_time)+'\n')
f.close()
del out_img
def do_kHighLevelEvent(self, event):
import AE
AE.AEProcessAppleEvent(event)
'batch_size': 256,
'shuffle': True,
'maxHits': 20,
'flatten': True,
'normalize': True
}
train_params = {
'epochs': 10,
'outdir': outdir,
'patience_count': 5,
'monitor': 'val_loss'
}
if (not os.path.isdir(outdir)): os.mkdir(outdir)
arch = AE(**model_params)
arch.buildModel()
inputFiles = glob.glob(generator_params['input_dir'] + 'images_*.root.npz')
inputIndices = np.array([f.split('images_')[-1][:-9] for f in inputFiles])
nFiles = len(inputIndices)
print('Found', nFiles, 'input files')
file_ids = {'train': inputIndices[:200], 'validation': inputIndices[500:550]}
train_generator = DataGenerator(file_ids['train'], **generator_params)
val_generator = DataGenerator(file_ids['validation'], **generator_params)
arch.fit_generator(train_generator=train_generator,
val_generator=val_generator,
**train_params)
verbose=1,save_best_only=True)
lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.1),
cooldown=0,
patience=5,
min_lr=0.5e-6)
lr_scheduler = LearningRateScheduler(lr_schedule)
model.fit(trDat,trLbl,validation_data=(tsDat,tsLbl),epochs=50,batch_size=32,shuffle=True,callbacks=[checkpoint,lr_reducer,lr_scheduler])
def load_weights(model):
model.load_weights('classifier_weights.h5')
return model
if(__name__=='__main__'):
classifier_model=createResnetV1((384,512,3))
classifier_model.load_weights('classifier_weights.h5')
ae=AE.create_model()
ae.load_weights('model_weights.h5')
for direc in ['test_images/true/','test_images/false/']:
images=read_images(direc)
for img in images:
imgplt(img)
if(AE.classify(ae,img.reshape(1,384,512,3))):
pred=classifier_model.predict(img.reshape(1,384,512,3))
pred=np.argmax(pred[0])
if(pred==0):
print('paper')
elif(pred==1):
print('plastic')
else:
print('can')
else:
def main():
appleid = 1
ClearMenuBar()
applemenu = NewMenu(appleid, "\024")
applemenu.AppendMenu("All about me...;(-")
applemenu.AddResMenu('DRVR')
applemenu.InsertMenu(0)
DrawMenuBar()
winwidth = 200
winheight = 100
r = (30, 60, 30 + winwidth, 60 + winheight)
w = NewWindow(r, "Hello world", 1, zoomDocProc, -1, 1, 0)
resizerect = (10, 20, 70, 40)
resize = NewControl(w, resizerect, "Resize", 1, 0, 0, 1, pushButProc, 0)
quitrect = (90, 20, 130, 40)
quit = NewControl(w, quitrect, "Quit", 1, 0, 0, 1, pushButProc, 0)
while 1:
doit, event = WaitNextEvent(-1, 0)
if doit:
SetPort(w)
(what, message, when, where, modifiers) = event
if what == mouseDown:
partcode, win = FindWindow(where)
if partcode == inDrag: # transvestites, take note
win.DragWindow(where, everywhere)
elif partcode == inGoAway:
if win.TrackGoAway(where):
break
elif partcode in (inZoomIn, inZoomOut):
SetPort(win) # !!!
if win.TrackBox(where, partcode):
win.ZoomWindow(partcode, 1)
elif partcode == inSysWindow:
print "SystemWindow"
elif partcode == inDesk:
print "Desk"
elif partcode == inMenuBar:
result = MenuSelect(where)
id = (result >> 16) & 0xffff
if id:
item = result & 0xffff
print "Menu id", id, ", item", item
if id == appleid:
name = applemenu.GetItem(item)
print name
OpenDeskAcc(name)
HiliteMenu(0)
elif partcode == inGrow:
result = win.GrowWindow(where, everywhere)
if result:
winheight = (result >> 16) & 0xffff
winwidth = result & 0xffff
win.SizeWindow(winwidth, winheight, 0)
SetPort(win)
InvalRect(everywhere)
elif partcode == inContent:
local = GlobalToLocal(where)
print "local =", local
ctltype, control = FindControl(local, win)
if ctltype and control:
pcode = control.TrackControl(local)
if pcode:
if control is resize:
if winwidth == 200:
winwidth = 300
else:
winwidth = 200
win.SizeWindow(winwidth, winheight, 0)
SetPort(win)
InvalRect(everywhere)
elif control is quit:
break
else:
print ctltype, control
else:
print "Mouse down at", where, "(global)"
print partcode, win
elif what == mouseUp:
print "Mouse up at", where, "(global)"
elif what == keyDown:
print "Key down:", hex(message), hex(modifiers)
char = chr(message & 0xff)
keycode = (message & 0xff00) >> 8
print 'char:', ` char `, '; keycode:', keycode
if char == 'Q': break
elif what == autoKey:
print "Key repeat:", hex(message), hex(modifiers)
elif what == updateEvt:
print "update Event", message, modifiers
w.BeginUpdate()
EraseRect(everywhere)
MoveTo(0, 0)
LineTo(100, 100)
TextSize(9)
DrawString("This is a dead parrot!")
MoveTo(0, 100)
LineTo(100, 0)
Move(0, 20)
TextSize(12)
DrawString("Nobody expects the Spanish Inquisition")
TextSize(20)
DrawString("!")
UpdateControls(w)
w.DrawGrowIcon()
w.EndUpdate()
elif what == activateEvt:
print "activateEvt:", message, modifiers
InitCursor()
elif what == osEvt:
print "osEvt:", message, where, modifiers
elif what == 23:
try:
AE.AEProcessAppleEvent(event)
except AE.Error, msg:
print "AE.Error:", msg
else:
print "???", event
import warnings
import sys
sys.path.insert(0, '../lib/pylib/')
from preprocess_10x import getNormAnnData
import AE
warnings.filterwarnings("ignore")
if sys.argv[2] == "log":
adata, size_factor = getNormAnnData(sys.argv[1], sys.argv[2])
AE.train_zinb_model(adata, size_factor)
AE.prediction_zinb(adata, size_factor)
else:
adata = getNormAnnData(sys.argv[1], sys.argv[2])
AE.train_zip_model(adata)
AE.prediction_zip(adata)
if sys.argv[1] == 'pca':
pca = PCA(n_components=DIM)
train_X_reduce = np.concatenate((pca.fit_transform(
np.array([x[0] + x[1]
for x in train_X])), np.array([x[2] for x in train_X])),
axis=1)
# Applies same model on test data.
test_X_reduce = np.concatenate(
(pca.transform(np.array([x[0] + x[1] for x in test_X
])), np.array([x[2] for x in test_X])),
axis=1)
print("Dimension reduction done with PCA")
else:
feature.standardizeX(train_X)
feature.standardizeX(test_X)
train_X_reduce, ae_w, ae_b = AE.dim_reduce(
np.array([x[0] + x[1] for x in train_X]), DIM, BATCH, 50, 0.01)
train_X_reduce = np.concatenate(
(train_X_reduce, np.array([x[2] for x in train_X])), axis=1)
# Applies same model on test data
test_X_reduce = np.concatenate(
(AE.forward2hidden(np.array([x[0] + x[1] for x in test_X]), ae_w,
ae_b, DIM), np.array([x[2] for x in test_X])),
axis=1)
print("Dimension reduction done with AE")
clf = svm.SVR(verbose=True)
clf.fit(train_X_reduce, train_Y)
print("Support vector machine fitting done.")
# Predict
migration_threshold = input('migration_threshold = ')
op = input('type 0 to use PCA, 1 to use AE : ')
f = Feature(node_threshold=180, migration_threshold=migration_threshold)
X, Y = f.getYearFeatures(2015)
X0 = []
X1 = []
for i in range(len(X)):
X0.append(X[i][0])
X1.append(X[i][1])
if op == 0:
x0 = DR_PCA(X0, dim)
x1 = DR_PCA(X1, dim)
else:
X0 = np.array(X0)
X1 = np.array(X1)
x, w, b = AE.dim_reduce(X0, dim, 2, 20, 0.01)
x0 = AE.forward2hidden(X0, w, b, 2)
x, w, b = AE.dim_reduce(X1, dim, 2, 20, 0.01)
x1 = AE.forward2hidden(X1, w, b, 2)
x = []
for i in range(len(X)):
x.append(np.concatenate((x0[i], x1[i], X[i][2])))
print ('Dimension reduction done.')
x = np.array(x)
dim = 2*dim+3
num = int(sqrt(len(x)))
O = [sum([Y[i*num+j] for j in range(num)]) for i in range(num)]
I = [sum([Y[i*num+j] for i in range(num)]) for j in range(num)]
data_driver.save_data("./data/save_dataDriver_actionPos_without_norm",0,data_driver)#adapt to record data_driver
data_driver.save_data("./data/save_my_data_actionPos_without_norm",0,my_data)
data_driver.save_data("./data/save_test_actionPos_without_norm",0,test)
else:
mnist = data_driver.read_data("./data/save_my_data_actionPos_without_norm",0)#_xpAdrienPos",0)
data_driver = data_driver.read_data("./data/save_dataDriver_actionPos_without_norm",0) # _xpAdrienPos",0)
test_data, test_labels = data_driver.read_data("./data/save_test_actionPos_without_norm",0)#_xpAdrienPos",0)
############### Fin copie ############################################################
ae = AE.AE(num_input,num_hidden_1,n_z,learning_rate,typeInit = typeInit, typeActivation=typeActivation,typeOpti=typeOpti,restore=flag_restore, epoch=training_epochs)
dt = DataTreatment
if(flag_restore == False): # If we don't restore a previous learning, then we train
for i in range(1, training_epochs+1):
# Prepare Data
# Get the next batch of MNIST data (only images are needed, not labels)
batch_x, _ = mnist.train.next_batch(batch_size)
# Run optimization op (backprop) and cost op (to get loss value)
l = ae.partial_fit(batch_x, i)
# Display logs per step
if i % display_step == 0 or i == 1:
print('Step %i: Minibatch Loss: %f' % (i, l))
ae.save_session(training_epochs,n_z)
data = np.array([[1, 2, 3, 4, 5, 1, 2, 3, 4, 5], [5, 2, 4, 1, 3, 5, 2, 4, 1, 3], [1, 2, 3, 4, 5, 1, 2, 3, 4, 5], [5, 2, 4, 1, 3, 5, 2, 4, 1, 3]])
print('input:', data.shape[0], ',', data.shape[1])
print(data)
print('==============================')
# API: dim_reduce (dataN, HIDDEN_N, batch_size=256, training_epochs=20, learning_rate=0.01):
################### Training data ############################
# data: (vectorN, dim of each vector) must be "normalized" before dim_reduce
# batch_size: since vectorN (i.e.data.shape[0]) may be very large, we devides them into batches and train each batch.
# e.g. data: (256000, 999) and we set batch_size to 256, we train data of size (256, 999) each time
#
################### Parameters to adjust ############################
# HIDDEN_N: #neuron in the hidden layer
# training_epochs: iteration
# learning_rate: :)
################### output ############################
# data after dim_reduce: (vectorN, HIDDEN_N)
# w
# b
X, w, b = AE.dim_reduce(data, 3, 2, 20, 0.01)
print('==============================')
print('answer:', X.shape[0], ',', X.shape[1])
print(X)
#forward2hidden(data, w, b, batch_size):
XX = AE.forward2hidden(data, w, b, 2)
print(XX)
