def worker(data, pname, que, lock, is_train, mean, std):
p = OutputPipe(pname, buffer_size=200)
worker_id = int(que.get())
que.put(worker_id + 1)
add_rand_seed_entropy(worker_id)
add_rand_seed_entropy(pname)
np.random.seed(stable_rand_seed(worker))
with control(io=[p]):
while True:
#np.random.set_state(que.get())
idx = np.random.randint(len(data[0]))
x, y = np.random.randint(9, size=2)
flag = np.random.randint(2)
#que.put(np.random.get_state())
#que.put((int(idx) + 1) % len(data[0]))
img = np.array(data[0][int(idx)])
img = img.astype(np.float32)
img = (img - mean) / std
img = np.resize(img, (3, 32, 32))
#img = (img - 128) / 256
if is_train:
img = augment(img, x, y)
if flag == 1:
img = img[:, :, ::-1]
print("reversed")
#a = msgpack.packb([img, data[1][int(idx)]], default = m.encode)
#b = msgpack.unpackb(a, object_hook = m.decode)
#print(np.array(b[0]).shape, b[1])
p.put_pyobj([np.array(img), int(data[1][int(idx)])])
print("put {} data {} successfully".format({
True: "train",
False: "valid"
}[is_train], int(idx)))
def worker(data, pname, que, lock, is_train, mean, std):
p = OutputPipe(pname, buffer_size=200)
worker_id = int(que.get())
que.put(worker_id + 1)
add_rand_seed_entropy(worker_id)
add_rand_seed_entropy(pname)
np.random.seed(stable_rand_seed(worker))
with control(io=[p]):
while True:
#np.random.set_state(que.get())
idx = np.random.randint(len(data[0]))
x, y = np.random.randint(9, size=2)
flag = np.random.randint(2)
#que.put(np.random.get_state())
#que.put((int(idx) + 1) % len(data[0]))
img = np.array(data[0][int(idx)])
img = img.astype(np.float32)
img = (img - mean) / std
img = np.resize(img, (3, 32, 32))
#img = (img - 128) / 256
if is_train:
img = augment(img, x, y)
if flag == 1:
img = img[:, :, ::-1]
print("reversed")
imgs = [img]
patch_size = 32
l = int(patch_size * 0.8)
s = [[0, l], [patch_size - l, patch_size]]
for i in range(4):
xx = s[i % 2]
yy = s[i // 2]
for j in range(3):
fmap = img[j]
fmap = fmap[xx[0]:xx[1], yy[0]:yy[1]]
fmap = cv2.resize(fmap, (patch_size, patch_size))
imgs.append([fmap])
img = np.concatenate(imgs, axis=0)
#print(img[0])
#print(img[3])
#a = msgpack.packb([img, data[1][int(idx)]], default = m.encode)
#b = msgpack.unpackb(a, object_hook = m.decode)
#print(np.array(b[0]).shape, b[1])
p.put_pyobj([np.array(img), int(data[1][int(idx)])])
print("put {} data {} successfully".format({
True: "train",
False: "valid"
}[is_train], int(idx)))
def worker():
#add_rand_seed_entropy(worker_id)
np.random.seed(int(time.time() * 100000 % 1000000))
fetcher = DataFetcher(is_train=True)
addr = datafeed_addr('train')
q = OutputPipe(addr, buffer_size=100)
with control(io=[q]):
logger.info('started worker')
while True:
img, label = fetcher.load(True)
img_aug = Augmentor.process(img)
# msgdata cannot serialize float32
label = int(label)
img_aug = img_aug.astype(np.uint8)
data = {'img': img_aug, 'label': [label]}
q.put_pyobj(data)
# print('putted')
sys.stdout.flush()
from dpflow import control, OutputPipe
q = OutputPipe("lyy-pipe-test", buffer_size = 100)
with control(io = [q]):
epoc = 0
while True:
q.put_pyobj(epoc)
print(epoc)
epoc += 1