# coding: utf-8

# ## Data Preparation

# In[1]:

get_ipython().magic('doctest_mode')

# In[2]:

import pandas

raw_data = pandas.read_csv("agaricus-lepiota.data", header=None)

split = 2031

data_one_hot = pandas.get_dummies(raw_data)

training_one_hot = (data_one_hot.iloc[split:, 2:], data_one_hot.iloc[split:, :2])

testing_one_hot = (data_one_hot.iloc[:split, 2:], data_one_hot.iloc[:split, :2])

from fuel.datasets import IndexableDataset

training_dataset = IndexableDataset(

indexables={'features': training_one_hot[0].values.astype('i8'), 'targets': training_one_hot[1].values.astype('i8')})

testing_dataset = IndexableDataset(

indexables={'features': testing_one_hot[0].values.astype('i8'), 'targets': testing_one_hot[1].values.astype('i8')})

# ## Blocks Tutorial

# In[3]:

import theano

# theano.config.optimizer = "None"

# theano.config.exception_verbosity = "high"

# In[4]:

from theano import tensor

x = tensor.lmatrix('features')

# In[5]:

from blocks.bricks import Linear, Logistic, Softmax

# In[10]:

hidden_layer_size = 100

input_to_hidden = Linear(name='input_to_hidden', input_dim=117, output_dim=hidden_layer_size)

h = Logistic().apply(input_to_hidden.apply(x))

hidden_to_output = Linear(name='hidden_to_output', input_dim=hidden_layer_size, output_dim=2)

y_hat = Softmax().apply(hidden_to_output.apply(h))

y = tensor.lmatrix('targets')

from blocks.bricks.cost import CategoricalCrossEntropy, MisclassificationRate

cost = CategoricalCrossEntropy().apply(y, y_hat)

error_rate = MisclassificationRate().apply(y.argmax(axis=1), y_hat)

error_rate.name = "error_rate"

# >>> from blocks.roles import WEIGHT

from blocks.graph import ComputationGraph

# >>> from blocks.filter import VariableFilter

cg = ComputationGraph(cost)

# >>> W1, W2 = VariableFilter(roles=[WEIGHT])(cg.variables)

# >>> cost = cost + 0.005 * (W1 ** 2).sum() + 0.005 * (W2 ** 2).sum()

# >>> cost.name = 'cost_with_regularization'

cost.name = 'cost_simple_xentropy'

from blocks.initialization import IsotropicGaussian, Constant

input_to_hidden.weights_init = hidden_to_output.weights_init = IsotropicGaussian(0.01)

input_to_hidden.biases_init = hidden_to_output.biases_init = Constant(0)

input_to_hidden.initialize()

hidden_to_output.initialize()

from fuel.streams import DataStream

from fuel.schemes import SequentialScheme, SequentialExampleScheme

# >>> from fuel.transformers import Flatten

data_stream = DataStream.default_stream(

training_dataset,

iteration_scheme=SequentialScheme(training_dataset.num_examples, batch_size=20))

data_stream_test = DataStream.default_stream(

testing_dataset,

iteration_scheme=SequentialScheme(testing_dataset.num_examples, batch_size=split))

from blocks.extensions.monitoring import DataStreamMonitoring

monitor = DataStreamMonitoring(

variables=[cost, error_rate], data_stream=data_stream_test, prefix="test")

# In[11]:

get_ipython().magic('pinfo DataStreamMonitoring')

# In[ ]:

# In[8]:

cost.type

# In[7]:

from blocks.algorithms import GradientDescent, Scale

algorithm = GradientDescent(cost=cost, parameters=cg.parameters,

step_rule=Scale(learning_rate=0.025))

from blocks.main_loop import MainLoop

from blocks.extensions import FinishAfter, Printing

from blocks_extras.extensions.plot import Plot

main_loop = MainLoop(data_stream=data_stream, algorithm=algorithm,

])

main_loop.run()

# ### Observations:

#

# **VERY IMPORTANT**

# there's some sort of shared state going on in the model definition, so it's important to rerun all the code from the beginning, not just the main loop!

#

# Setting the hidden layer to 50 lowered the cost (0.69), but didn't improve the score after more training.

# Setting the hidden layer to 300 bumped up the cost (1.79), but training made significant improvements in it after first epoch, but not in subsequent ones (more incremental after that).

#

# Somehow the total number of epochs influences the starting cost??

#

# - epochs: 5 vs 3

# - learning rate: 0.5

# - hidden layer: 300

#

# When I took the same parameters (5 epochs) and set hidden layer to 100, I got the following progression of costs:

#

# - epochs done: 0 = 0.6931921183574188

# - epochs done: 1 = 1.814269964941275

# - epochs done: 2 = 1.194301165186615

# - epochs done: 3 = 0.8182568883881371

# - epochs done: 4 = 0.7323559855023634

# - epochs done: 5 = 0.6993469372860405

#

#

# ### Data Processing

#

# What is `Flatten` for?

#

# ### Minibatches and Train/Test Split

#

# What's the relationship between the test/training data and the minibatch size?

# Does the batch size have to "fit" exactly into the dataset sizes?

# Why are we also iterating over the test data?

#

#

# ## From Theano intro tutorial

# In[1]:

import theano

from theano import tensoror

# In[7]:

a = tensor.dscalar("a")

b = tensor.dscalar("b")

# In[8]:

c = a + b

f = theano.function([a, b], c)

# In[4]:

assert 4 == f(1.5, 2.5)

# In[9]:

theano.pp(c)

# In[16]:

c.owner.op.name