def worker(data, pname, que, lock, is_train, mean, std):
p = OutputPipe(pname, buffer_size=200)
worker_id = int(que.get())
que.put(worker_id + 1)
add_rand_seed_entropy(worker_id)
add_rand_seed_entropy(pname)
np.random.seed(stable_rand_seed(worker))
with control(io=[p]):
while True:
#np.random.set_state(que.get())
idx = np.random.randint(len(data[0]))
x, y = np.random.randint(9, size=2)
flag = np.random.randint(2)
#que.put(np.random.get_state())
#que.put((int(idx) + 1) % len(data[0]))
img = np.array(data[0][int(idx)])
img = img.astype(np.float32)
img = (img - mean) / std
img = np.resize(img, (3, 32, 32))
#img = (img - 128) / 256
if is_train:
img = augment(img, x, y)
if flag == 1:
img = img[:, :, ::-1]
print("reversed")
#a = msgpack.packb([img, data[1][int(idx)]], default = m.encode)
#b = msgpack.unpackb(a, object_hook = m.decode)
#print(np.array(b[0]).shape, b[1])
p.put_pyobj([np.array(img), int(data[1][int(idx)])])
print("put {} data {} successfully".format({
True: "train",
False: "valid"
}[is_train], int(idx)))
def _get_dataset_mean_var(data, func, i):
total_mean = None
total_Esqr = None
p = InputPipe("lyy.CIFAR10.train", buffer_size = 1000)
with control(io = [p]):
data = get_minibatch(p, 30)
output = func(data = data["data"])
total_mean = output[int(i) * 2].mean(axis = 0)
total_Esqr = output[int(i) * 2 + 1].mean(axis = 0)
return total_mean, total_Esqr - total_mean**2
def worker(data, pname, que, lock, is_train, mean, std):
p = OutputPipe(pname, buffer_size=200)
worker_id = int(que.get())
que.put(worker_id + 1)
add_rand_seed_entropy(worker_id)
add_rand_seed_entropy(pname)
np.random.seed(stable_rand_seed(worker))
with control(io=[p]):
while True:
#np.random.set_state(que.get())
idx = np.random.randint(len(data[0]))
x, y = np.random.randint(9, size=2)
flag = np.random.randint(2)
#que.put(np.random.get_state())
#que.put((int(idx) + 1) % len(data[0]))
img = np.array(data[0][int(idx)])
img = img.astype(np.float32)
img = (img - mean) / std
img = np.resize(img, (3, 32, 32))
#img = (img - 128) / 256
if is_train:
img = augment(img, x, y)
if flag == 1:
img = img[:, :, ::-1]
print("reversed")
imgs = [img]
patch_size = 32
l = int(patch_size * 0.8)
s = [[0, l], [patch_size - l, patch_size]]
for i in range(4):
xx = s[i % 2]
yy = s[i // 2]
for j in range(3):
fmap = img[j]
fmap = fmap[xx[0]:xx[1], yy[0]:yy[1]]
fmap = cv2.resize(fmap, (patch_size, patch_size))
imgs.append([fmap])
img = np.concatenate(imgs, axis=0)
#print(img[0])
#print(img[3])
#a = msgpack.packb([img, data[1][int(idx)]], default = m.encode)
#b = msgpack.unpackb(a, object_hook = m.decode)
#print(np.array(b[0]).shape, b[1])
p.put_pyobj([np.array(img), int(data[1][int(idx)])])
print("put {} data {} successfully".format({
True: "train",
False: "valid"
}[is_train], int(idx)))
def worker():
#add_rand_seed_entropy(worker_id)
np.random.seed(int(time.time() * 100000 % 1000000))
fetcher = DataFetcher(is_train=True)
addr = datafeed_addr('train')
q = OutputPipe(addr, buffer_size=100)
with control(io=[q]):
logger.info('started worker')
while True:
img, label = fetcher.load(True)
img_aug = Augmentor.process(img)
# msgdata cannot serialize float32
label = int(label)
img_aug = img_aug.astype(np.uint8)
data = {'img': img_aug, 'label': [label]}
q.put_pyobj(data)
# print('putted')
sys.stdout.flush()
train_func.optimizer_state)
tr_p = InputPipe("lyy.CIFAR10.train", buffer_size=1000)
va_p = InputPipe("lyy.CIFAR10.valid", buffer_size=1000)
epoch = 0
EPOCH_NUM = 49500 // minibatch_size
i = 0
max_acc = 0
ORI_IT = 64000
BN_IT = 10000
TOT_IT = ORI_IT + BN_IT
his = []
his_test = []
import time
with control(io=[tr_p]):
with control(io=[va_p]):
a = time.time()
while i <= TOT_IT:
i += 1
token1 = time.time()
data = get_minibatch(tr_p, minibatch_size)
time_data = time.time() - token1
token2 = time.time()
out = train_func(data=data['data'], label=data["label"])
time_train = time.time() - token2
if time_data > (time_train + time_data) * 0.1:
print(time_data / (time_train + time_data))
from dpflow import control, OutputPipe
q = OutputPipe("lyy-pipe-test", buffer_size = 100)
with control(io = [q]):
epoc = 0
while True:
q.put_pyobj(epoc)
print(epoc)
epoc += 1
from dpflow import InputPipe, control
p = InputPipe("lyy-pipe-test", buffer_size = 100)
with control(io = [p]):
for i in range(100):
print(p.get())