np.random.seed(0) # determinism
fn = os.path.join(settings['data-base'], 'positions.csv')
reader = csv.reader(open(fn), dialect='unix')
reader.__next__() # skip headers
print("preparing to read data...")
# nolines = 3977256
nolines = 1800000
# nolines = 30000
feats = []
labels = []
print("reading data...")
for i, row in enumerate(reader):
feats.extend(featdecode(row[4]))
labels.append([lint(row[3])])
if i + 1 >= nolines:
break
if i % 100000 == 0:
print(i)
print("reshaping data...")
samples = len(feats) // 384
X_all = np.array(feats).reshape((samples, 6, 8, 8))
y_all = np.array(labels)
##################################################################################
for (dropout_rate, regularize, learning_rule), kernel_shape in \
print("loading model...")
fn = os.path.join(settings['data-base'], 'nn.pickle')
mlp = pickle.load(open(fn, 'rb'))
fn = os.path.join(settings['data-base'], 'positions.csv')
reader = csv.reader(open(fn), dialect='unix')
reader.__next__() # skip headers
print("preparing to read data...")
nolines = 3977256
# nolines = 1000
feats = collections.defaultdict(list)
labels = collections.defaultdict(list)
print("reading data...")
for h, row in enumerate(reader):
feats[int(row[2])].extend(featdecode(row[4]))
labels[int(row[2])].append([lint(row[3])])
if h + 1 >= nolines:
break
if h % 100000 == 0:
print(h)
print("reshaping data...")
Xs = dict()
ys = dict()
for h in feats:
samples = len(feats[h]) // 384
Xs[h] = np.array(feats[h]).reshape((samples, 6, 8, 8))
ys[h] = np.array(labels[h])
mse_scores = dict()
learning_rule='nesterov',
verbose=True
)
fn = os.path.join(settings['data-base'], 'positions.csv')
reader = csv.reader(open(fn), dialect='unix')
reader.__next__() # skip headers
print("preparing to read data...")
nolines = 3977256
# nolines = 400000
feats = []
labels = []
print("reading data...")
for i, row in enumerate(reader):
feats.extend(featdecode(row[4]))
labels.append([lint(row[3])])
if i + 1 >= nolines:
break
if i % 100000 == 0:
print(i)
print("reshaping data...")
samples = len(feats) // 384
X = np.array(feats).reshape((samples, 6, 8, 8))
y = np.array(labels)
print("shuffling data...")
examples = list(zip(X, y))
X, y = list(zip(*examples))