def test_get_full_path_windows(self):
folder_name = '/utils/tests'
file_name = 'test.txt'
expected = os.getcwd() + '\\test.txt'
if platform.system() == 'Linux':
expected = [
'/home/travis/build/ikostan/'
'ProjectEuler/utils/tests/test.txt',
'/home/circleci/project/utils/tests/test.txt'
]
self.assertTrue(get_full_path(folder_name, file_name) in expected)
def plotTimings(figIdx, dataset, e_layer_sizes, e_times, e_times_all, e_epochs,
c_layer_sizes, c_times, c_epochs):
plt.figure(figIdx)
plt.subplot(3, 1, 1)
plt.title("%s - Timing" % dataset)
# (e_lpre, c_lpre, e_lrec, c_lrec, e_lf1, c_lf1) = plt.plot(e_layer_sizes, e_pres, 'wo', c_layer_sizes, c_pres, 'ko', e_layer_sizes, e_recs, 'w^', c_layer_sizes, c_recs, 'k^', e_layer_sizes, e_scores_f1, 'b.', c_layer_sizes, c_scores_f1, 'r.')
(e_ltimes, c_ltimes) = plt.plot(e_layer_sizes, e_times, 'r', c_layer_sizes,
c_times, 'b')
plt.setp(e_ltimes, label='Elastic')
plt.setp(c_ltimes, label='Compound')
plt.ylabel("Training Time [s]")
plt.grid(True)
plt.legend(loc='upper left', numpoints=1)
plt.subplot(3, 1, 2)
(e_lepochs, c_lepochs) = plt.plot(e_layer_sizes, e_epochs, 'r',
c_layer_sizes, c_epochs, 'b')
plt.setp(e_lepochs, label='Elastic')
plt.setp(c_lepochs, label='Compound')
plt.ylabel("Epochs")
plt.grid(True)
plt.legend(loc='upper left', numpoints=1)
plt.savefig(
utils.get_full_path(Config.PATH_EVAL_ROOT, "%s-timing.pdf" % dataset))
plt.subplot(3, 1, 3)
(e_ltimepe,
c_ltimepe) = plt.plot(e_layer_sizes, [np.mean(x) for x in e_times_all],
'r', c_layer_sizes,
np.asarray(c_times) / np.asarray(c_epochs), 'b')
plt.setp(e_ltimepe, label='Elastic')
plt.setp(c_ltimepe, label='Compound')
plt.ylabel("Time per epoch [s]")
plt.grid(True)
plt.legend(loc='upper left', numpoints=1)
plt.xlabel('Layer Size')
plt.savefig(
utils.get_full_path(Config.PATH_EVAL_ROOT, "%s-timing.pdf" % dataset))
def words_reader(file_folder: str, file_name: str):
"""
Reads file and returns all words as a list
:param file_folder:
:param file_name:
:return:
"""
results = list()
path = get_full_path(file_folder, file_name)
with open(path, 'r') as reader:
for line in reader:
results += [l.strip('"') for l in line.split(',')]
return results
def plotErrorsSep(figIdx, dataset, e_errors_sep, c_errors_sep):
plt.figure(figIdx)
plt.title("%s - Error (per layer size)" % dataset)
for idx, errors in enumerate(e_errors_sep):
ax = plt.subplot(3, 10, idx)
plt.plot(e_errors_sep[idx], 'r', c_errors_sep[idx], 'b')
ax.get_xaxis().set_ticks([])
ax.get_yaxis().set_ticks([])
plt.grid(True)
plt.yticks()
plt.tight_layout()
plt.savefig(
utils.get_full_path(Config.PATH_EVAL_ROOT,
"%s-error-sep.pdf" % dataset))
def setUpClass(cls):
"""
Get all leap years from the text file
and organize them as an array
:return:
"""
cls.leap_years = []
file_path = utils.get_full_path('/Problems_1_to_100/Problem_19/tests',
'leap_years.txt')
with open(file_path) as source:
for line in source:
cls.leap_years.append(int(line.strip()))
def plotErrors(figIdx, dataset, e_layer_sizes, e_errors, c_layer_sizes,
c_errors):
plt.figure(figIdx)
plt.title("%s - Error" % dataset)
# (e_lpre, c_lpre, e_lrec, c_lrec, e_lf1, c_lf1) = plt.plot(e_layer_sizes, e_pres, 'wo', c_layer_sizes, c_pres, 'ko', e_layer_sizes, e_recs, 'w^', c_layer_sizes, c_recs, 'k^', e_layer_sizes, e_scores_f1, 'b.', c_layer_sizes, c_scores_f1, 'r.')
(e_lerr, c_lerr) = plt.plot(e_layer_sizes, e_errors, 'r', c_layer_sizes,
c_errors, 'b')
plt.setp(e_lerr, label='Elastic')
plt.setp(c_lerr, label='Compound')
plt.ylabel("Error")
plt.grid(True)
plt.legend(loc='upper right', numpoints=1)
plt.xlabel('Layer Size')
plt.savefig(
utils.get_full_path(Config.PATH_EVAL_ROOT, "%s-error.pdf" % dataset))
def test_get_full_path_linux(self):
target_os = 'Linux'
folder_name = '/utils/tests'
file_name = 'test.txt'
expected = [
'/home/travis/build/ikostan/'
'ProjectEuler/utils/tests/test.txt',
'/home/circleci/project/utils/tests/test.txt'
]
if platform.system() == 'Windows':
expected = os.getcwd() + '/utils/tests' + '/test.txt'
self.assertTrue(
get_full_path(folder_name, file_name, target_os) in expected)
def test_compound(hidden_size):
mode_name = "compound"
total_logs = {}
for i in range(0, hidden_size, 10):
s = i+1
id = (mode_name + "-" + data_name + "-" + str(s))
ae = Autoencoder(nvis=len(train_data[0][0]), nhid=s, corruption_level=corruption_level, log_level=log_level, caching=False)
strt = time.time()
logs = ae.train(train_data[0], elastic=False, epochs=None, mini_batch_size=mini_batch_size, stop_err_delta=stop_err_delta, supervisedDataCallback=func)
end = time.time()
print "Total Training Time: %.3f" % (end-strt)
total_logs[str(i)] = logs[str(i)]
ae.visualize_filters("GREY", name=id)
ae.dump_model(str(id))
cPickle.dump(total_logs, open(utils.get_full_path(Config.PATH_EVAL_ROOT, "results-%s.pkl" % id), 'wb'))
return total_logs
def plotScores(figIdx, dataset, e_layer_sizes, e_pres, e_recs, e_scores_f1,
c_layer_sizes, c_pres, c_recs, c_scores_f1):
plt.figure(figIdx)
plt.subplot(3, 1, 1)
plt.title("%s - Classification (Mean)" % dataset)
# (e_lpre, c_lpre, e_lrec, c_lrec, e_lf1, c_lf1) = plt.plot(e_layer_sizes, e_pres, 'wo', c_layer_sizes, c_pres, 'ko', e_layer_sizes, e_recs, 'w^', c_layer_sizes, c_recs, 'k^', e_layer_sizes, e_scores_f1, 'b.', c_layer_sizes, c_scores_f1, 'r.')
(e_lpre, c_lpre) = plt.plot(e_layer_sizes, e_pres, 'ro', c_layer_sizes,
c_pres, 'bo')
plt.setp(e_lpre, label='Elastic')
plt.setp(c_lpre, label='Compound')
plt.ylabel("Precision")
plt.grid(True)
plt.legend(loc='lower right', numpoints=1)
plt.subplot(3, 1, 2)
(e_lrec, c_lrec) = plt.plot(e_layer_sizes, e_recs, 'r^', c_layer_sizes,
c_recs, 'b^')
plt.setp(e_lrec, label='Elastic')
plt.setp(c_lrec, label='Compound')
plt.ylabel("Recall")
plt.grid(True)
plt.legend(loc='lower right', numpoints=1)
plt.subplot(3, 1, 3)
e_rects = plt.bar(np.asarray(e_layer_sizes),
e_scores_f1,
3,
alpha=0.8,
color='r',
error_kw={'ecolor': '0.3'},
label='Elastic')
c_rects = plt.bar(np.asarray(c_layer_sizes) + 3,
c_scores_f1,
3,
alpha=0.8,
color='b',
error_kw={'ecolor': '0.3'},
label='Compound')
plt.ylabel("F1")
plt.grid(True)
plt.legend(loc='lower right', numpoints=1)
plt.xlabel('Layer Size')
plt.savefig(
utils.get_full_path(Config.PATH_EVAL_ROOT, "%s-scores.pdf" % dataset))
def index(request, template_name):
if request.method == 'POST':
postdata = request.POST.copy()
form = RatingUserForm(postdata)
if form.is_valid():
key_expires = datetime.datetime.today() + datetime.timedelta(30)
user = form.save(commit=False)
if user.email == settings.KIOSK_EMAIL:
user.activation_key = 'letmein'
else:
user.activation_key = utils.activation_key(user)
user.ip = request.META['REMOTE_ADDR']
user.key_expires = key_expires
form.save()
if user.email == settings.KIOSK_EMAIL:
request.session['email'] = settings.KIOSK_EMAIL
request.session['key'] = 'letmein'
return redirect(urlresolvers.reverse('films'))
else:
html_template = loader.get_template(
'ratinguser/email_notification.html')
plain_template = loader.get_template(
'ratinguser/email_notification.txt')
context = Context({
'instance': user,
'url': utils.get_full_path(),
})
subject, from_email, to = "[Cinefest Audience Choice] Activation required", settings.FROM_EMAIL, user.email
html_content = html_template.render(context)
plain_content = plain_template.render(context)
msg = EmailMultiAlternatives(subject, plain_content,
from_email, [to])
msg.attach_alternative(html_content, "text/html")
msg.send()
messages.success(
request,
"Thank you. We've sent a verification link to your email address."
)
return redirect(urlresolvers.reverse('index'))
else:
form = RatingUserForm()
return render(request, template_name, {'form': form})
def file_reader():
"""
Use readline to read txt file python3:
Source: https://stackoverflow.com/questions/
28936140/use-readline-to-read-txt-file-python3
How to get the current working directory using python3:
https://stackoverflow.com/questions/17359698/
how-to-get-the-current-working-directory-using-python-3
:return:
"""
result = []
file_path = get_full_path('/Problems_1_to_100/Problem_22/tests',
'p022_names.txt')
with open(file_path) as source:
for line in source:
result = [t.replace('"', '') for t in line.split(',')]
return sorted(result)
def file_reader():
"""
Use readline to read txt file python3:
Source: https://stackoverflow.com/questions/
28936140/use-readline-to-read-txt-file-python3
How to get the current working directory using python3:
https://stackoverflow.com/questions/17359698/
how-to-get-the-current-working-directory-using-python-3
:return:
"""
result = []
file_path = get_full_path('/Problems_1_to_100/Problem_67/tests',
'p067_triangle.txt')
with open(file_path) as source:
for line in source:
temp = [int(t) for t in line.split(' ')]
# print(temp) # debug only
result.append(temp)
return result
def test_elastic(hidden_size):
mode_name = "elastic"
id = (mode_name + "-" + data_name + "-" + str(hidden_size))
ae = Autoencoder(nvis=len(train_data[0][0]), nhid=hidden_size, corruption_level=corruption_level, log_level=log_level, caching=True)
strt = time.time()
logs = ae.train(train_data[0], elastic=True, epochs=None, mini_batch_size=mini_batch_size, stop_err_delta=stop_err_delta, supervisedDataCallback=func)
end = time.time()
print "Total Training Time: %.3f" % (end-strt)
ae.visualize_filters("GREY", name=id)
# codes = ae.encode_all(train_data)
# codes[codes>0.] = 1.
# codes[codes<=0.] = 0.
# codes = codes.astype(int)
# intCodes = [utils.bool2int(x) for x in codes]
# print intCodes
# for i in range(2**hidden_size):
# print "Code %d" % i
# userIds = [idx for (idx, v) in enumerate(intCodes) if v == i]
# for user in userIds:
# print dataset.users.getUser(user)
ae.dump_model(id)
cPickle.dump(logs, open(utils.get_full_path(Config.PATH_EVAL_ROOT, "results-%s.pkl" % id), 'wb'))
return logs
def visualize_filters(self, mode, panel_shape=None, filter_shape=None, name=None):
"""
Plot weight matrix in a grid of shape ``panel_shape`` with each cell showing the weights of a hidden node,
shaped as given by ``filter_shape``.
:param mode: Either "RGB" or "GREY"
:param panel_shape:
:param filter_shape:
:param name:
:return:
"""
num_filters = self.W.shape[0]
num_features = self.W.shape[1]
if panel_shape is None:
panel_shape = (np.ceil(num_filters / 10.).astype(int), 10 + num_filters % 10)
if filter_shape is None:
filter_shape = (int(np.sqrt(num_features)), int(np.sqrt(num_features)))
img = utils.visualize_weights(self.W, panel_shape, filter_shape, mode)
if name is not None:
img.save(utils.get_full_path(Config.PATH_EVAL_ROOT, name + "-filters.png"), "PNG")
img.show()
ax.get_yaxis().set_ticks([])
plt.grid(True)
plt.yticks()
plt.tight_layout()
plt.savefig(
utils.get_full_path(Config.PATH_EVAL_ROOT,
"%s-error-sep.pdf" % dataset))
if __name__ == "__main__":
dataset = "MNIST"
maxSize = 301
fileName = utils.get_full_path(
Config.PATH_EVAL_ROOT, dataset, "elastic",
"results-elastic-" + dataset + "-" + str(maxSize) + ".pkl")
obj = pickleFromWindowsEncoding(fileName)
(layer_sizes, scores_f1, pres, recs, errors_sep, times, times_integrated,
times_all, times_overall) = extractInfo(obj)
c_fileName = utils.get_full_path(
Config.PATH_EVAL_ROOT, dataset, "compound",
"results-compound-" + dataset + "-" + str(maxSize) + ".pkl")
c_obj = pickleFromWindowsEncoding(c_fileName)
(c_layer_sizes, c_scores_f1, c_pres, c_recs, c_errors_sep, c_times,
c_times_integrated, c_times_all, c_times_overall) = extractInfo(c_obj)
epochs = [len(x) for x in times_all]
errors = [x[-1] for x in errors_sep]
c_epochs = [len(x) for x in c_times_all]
self._gradCache = [None] * len(self._outNodes)
self._instantiated = True
def f(self, x, y):
a = np.dot(self.W, x) + self.b
(cost, o, dEda) = self.OutModel.cost_out_grad(a, y)
return (cost, o)
def df(self, x, y):
a = np.dot(self.W, x) + self.b
(cost, o, dEda) = self.OutModel.cost_out_grad(a, y)
dEdx = np.dot(self.W.T, dEda)
return dEdx
def __str__(self):
return "%s\nOutput Model = %s\nW = %s\nb = %s" % (
Node.__str__(self), self.OutModel, self.W, self.b)
if __name__ == "__main__":
x = np.asarray([-1.0, 1.0, 0.5])
y = np.asarray([1., 0.])
cm = ConnectivityMatrix(
utils.get_full_path(Config.PATH_DATA_ROOT, Config.CONNECTIVITY_FILE))
g = Graph(3, 2, cm)
print g
print "propagation"
g.propagate(x, y)
cm.showAsImage()
g.backpropagate(x, y)
def load_model(name):
return cPickle.load(open(utils.get_full_path(Config.PATH_EVAL_ROOT, r"%s.pkl" % name), "r"))
def dump_model(self, name):
cPickle.dump(self, open(utils.get_full_path(Config.PATH_EVAL_ROOT, r"%s.pkl" % name), "w"))
self.b = np.zeros(shape=(self._dim_out,), dtype=theano.config.floatX)
self._inCache = [None] * len(self._inNodes)
self._gradCache = [None] * len(self._outNodes)
self._instantiated = True
def f(self, x, y):
a = np.dot(self.W, x) + self.b
(cost, o, dEda) = self.OutModel.cost_out_grad(a, y)
return (cost, o)
def df(self, x, y):
a = np.dot(self.W, x) + self.b
(cost, o, dEda) = self.OutModel.cost_out_grad(a, y)
dEdx = np.dot(self.W.T, dEda)
return dEdx
def __str__(self):
return "%s\nOutput Model = %s\nW = %s\nb = %s" % (Node.__str__(self), self.OutModel, self.W, self.b)
if __name__ == "__main__":
x = np.asarray([-1.0, 1.0, 0.5])
y = np.asarray([1., 0.])
cm = ConnectivityMatrix(utils.get_full_path(Config.PATH_DATA_ROOT, Config.CONNECTIVITY_FILE))
g = Graph(3, 2, cm)
print g
print "propagation"
g.propagate(x, y)
cm.showAsImage()
g.backpropagate(x, y)
def compareElasticCompound(self, elastic_logs, compound_logs):
e_scores_f1, c_scores_f1 = [], []
e_pres, c_pres = [], []
e_recs, c_recs = [], []
e_times, c_times = [], []
e_times_integrated = [0.]
e_errors, c_errors = [], []
e_errors_sep, c_errors_sep = [], []
e_last_errors, c_last_errors = [], []
e_tmp_list = []
for k in elastic_logs:
if representsInt(k):
e_tmp_list.append(int(k))
c_tmp_list = []
for k in compound_logs:
if representsInt(k):
c_tmp_list.append(int(k))
e_num_tests = max(e_tmp_list)
c_num_tests = max(c_tmp_list)
for numNodes in range(1, e_num_tests + 1):
(pre, rec,
f1) = elastic_logs[str(numNodes)][Autoencoder.STR_SCORES]
time = elastic_logs[str(numNodes)][Autoencoder.STR_TIMES]
error = elastic_logs[str(numNodes)][Autoencoder.STR_ERRORS]
e_scores_f1.append(np.mean(f1))
e_pres.append(np.mean(pre))
e_recs.append(np.mean(rec))
e_errors += error
e_errors_sep.append(error)
e_last_errors.append(error[-1])
e_times += time
e_times_integrated.append(e_times_integrated[-1] + sum(time))
for numNodes in range(1, c_num_tests + 1):
(pre, rec,
f1) = compound_logs[str(numNodes)][Autoencoder.STR_SCORES]
time = compound_logs[str(numNodes)][Autoencoder.STR_TIMES]
error = compound_logs[str(numNodes)][Autoencoder.STR_ERRORS]
c_scores_f1.append(np.mean(f1))
c_pres.append(np.mean(pre))
c_recs.append(np.mean(rec))
c_errors += error
c_errors_sep.append(error)
c_last_errors.append(error[-1])
c_times += time
plt.figure(1)
(e_lpre, c_lpre, e_lrec, c_lrec, e_lf1,
c_lf1) = plt.plot(e_pres, 'wo', c_pres, 'ko', e_recs, 'w^', c_recs,
'k^', e_scores_f1, 'b.', c_scores_f1, 'r.')
plt.setp(e_lpre, label='Precision (E)')
plt.setp(c_lpre, label='Precision (C)')
plt.setp(e_lrec, label='Recall (E)')
plt.setp(c_lrec, label='Recall (C)')
plt.setp(e_lf1, label='F1 (E)')
plt.setp(c_lf1, label='F1 (C)')
plt.axis([0, 100, 0, 1])
plt.grid(True)
plt.title("MNIST 10-Digit Classification")
plt.xlabel('Hidden Nodes')
plt.ylabel('Mean Classification Scores')
plt.legend(loc='lower right')
plt.savefig(utils.get_full_path(Config.PATH_EVAL_ROOT, "f1.pdf"))
plt.figure(2)
print len(e_last_errors)
print len(c_last_errors)
print e_last_errors
print c_last_errors
plt.step(range(1, e_num_tests + 1), e_last_errors, 'b')
plt.step(range(1, c_num_tests + 1), c_last_errors, 'r')
# plt.step(range(1,min((e_num_tests, c_num_tests))), c_last_errors)
plt.grid(True)
plt.legend(("Elastic", "Compound"))
plt.xlabel('Hidden Layer Size')
plt.ylabel('Reconstruction Error')
plt.savefig(utils.get_full_path(Config.PATH_EVAL_ROOT, "errors.pdf"))
plt.figure(3, figsize=(20, 6))
ax = plt.subplot(2, 1, 1)
summer = 0
ticks = [summer]
for (i, e) in enumerate(e_errors_sep):
plt.plot(range(summer, summer + len(e)), e, 'b')
# plt.plot((summer, summer), (0, 250), 'k-', linewidth=0.5, alpha=0.2)
summer += len(e)
ticks.append(summer)
plt.grid(True)
# Set the ticks and labels...
labels = []
for i in range(1, len(ticks) + 1):
if i % 10 == 0:
labels.append(str(i))
else:
labels.append(" ")
plt.xticks(ticks, labels)
plt.legend(("Elastic", ))
plt.xlabel('Hidden Layer Size')
plt.ylabel('Reconstruction Error')
ax = plt.subplot(2, 1, 2)
summer = 0
ticks = [summer]
for (i, e) in enumerate(c_errors_sep):
plt.plot(range(summer, summer + len(e)), e, 'r')
# plt.plot((summer, summer), (0, 250), 'k-', linewidth=0.5, alpha=0.2)
summer += len(e)
ticks.append(summer)
plt.grid(True)
# Set the ticks and labels...
labels = []
for i in range(1, len(ticks) + 1):
if i % 10 == 0:
labels.append(str(i))
else:
labels.append(" ")
plt.xticks(ticks, labels)
plt.legend(("Compound", ))
plt.xlabel('Hidden Layer Size')
plt.ylabel('Reconstruction Error')
plt.savefig(
utils.get_full_path(Config.PATH_EVAL_ROOT, "error-drops.pdf"))
plt.figure(4)
plt.subplot(2, 1, 1)
plt.plot(e_times)
plt.plot(e_times, 'bo')
plt.grid(True)
plt.title("Elastic")
plt.xlabel('Hidden Layer Size')
plt.ylabel('Training Time [s] (per node)')
plt.subplot(2, 1, 2)
plt.plot(e_times_integrated, 'b_', c_times, 'r_')
plt.grid(True)
plt.legend(("Elastic", "Compound"), loc='upper left')
plt.xlabel('Hidden Layer Size')
plt.ylabel('Training Time [s] (total)')
plt.savefig(utils.get_full_path(Config.PATH_EVAL_ROOT, "times.pdf"))
plt.show()
