def run_job(spark, config):
"""
This Function will execute the Analytics job 1
:param spark: SparkSession Object
:param config: Config file which contain all the parameters
:return: None
"""
load_data(analyse_data(spark, extract_data(spark, f"{config.get('source_data_path')}/Primary_Person_use.csv")),f"{config.get('target_data_path')}/job01")
def test_dA(dataset,
numFeatures,
numHidden,
minCorruptionLevel=0.0,
maxCorruptionLevel=1.0,
corruptionLevelStep=0.1,
learning_rate=0.1,
batch_size=20,
output_folder='dA_plots',
deltaCostStopThreshold=0.001,
learningRateBoostFactor=1.5):
'''
Inputs:
:dataset (str): path to the dataset in .pkl.gz format
:numFeatures: number of input features
:numHidden: number of hidden features
:learning_rate (float): learning rate used for training the DeNosing
AutoEncoder
:batch_size (int): number of examples to use in each batch
:output_folder (str): where to put the weights and biases
:deltaCostStopThreshold (float): string
:learningRateBoostFactor: a factor to multiply the learning rate by when
deltaCostStopThreshold is not met, use None to not use boosting
'''
datasets = load_data(dataset)
train_set_x, train_set_y = datasets[0]
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size
# allocate symbolic variables for the data
index = T.lscalar() # index to a [mini]batch
x = T.matrix('x') # the data is presented as rasterized images
# Create output folder if it doesn't exist
if not os.path.isdir(output_folder):
os.makedirs(output_folder)
os.chdir(output_folder)
origLearningRate = learning_rate
for corruptionLevel in numpy.arange(
minCorruptionLevel, maxCorruptionLevel + corruptionLevelStep,
corruptionLevelStep):
strCorruptionLevel = str(int(100 * corruptionLevel))
learning_rate = origLearningRate
######################
# BUILDING THE MODEL #
######################
rng = numpy.random.RandomState(123)
theano_rng = RandomStreams(rng.randint(2**30))
da = dA(numpy_rng=rng,
theano_rng=theano_rng,
input=x,
n_visible=numFeatures,
n_hidden=numHidden)
cost, updates = da.get_cost_updates(corruption_level=corruptionLevel,
learning_rate=learning_rate)
train_da = theano.function(
[index],
cost,
updates=updates,
givens={
x: train_set_x[index * batch_size:(index + 1) * batch_size]
})
start_time = time.clock()
############
# TRAINING #
############
# Go through training epochs until stopping criterion is met
deltaCost = -1
epoch = 0
lastEpochCost = 1e6
epochCost = 1e6
while deltaCost < -deltaCostStopThreshold or deltaCost < 0:
# Go through training set
c = []
weights = da.W.get_value(borrow=True).T
biases = da.b.get_value(borrow=True)
lastEpochCost = epochCost
for batch_index in xrange(n_train_batches):
c.append(train_da(batch_index))
epochCost = numpy.mean(c)
print 'Training epoch %d, cost ' % epoch, epochCost
deltaCost = epochCost - lastEpochCost
epoch += 1
# Check stopping criteria and update learning rate
if abs(deltaCost) < deltaCostStopThreshold:
learning_rate *= learningRateBoostFactor
print 'increasing learning rate to %0.2f...' % learning_rate
cost, updates = da.get_cost_updates(
corruption_level=corruptionLevel,
learning_rate=learning_rate)
train_da = theano.function(
[index],
cost,
updates=updates,
givens={
x: train_set_x[index * batch_size:(index + 1) *
batch_size]
})
# Save final weights and biases
outputFnRoot = 'corruption_%s_nin_%s_nhdn_%s_basz_%s_lnrt_%s' \
% (strCorruptionLevel, str(numFeatures), str(numHidden), str(batch_size), str(origLearningRate))
numpy.savetxt('weights_' + outputFnRoot + '.csv',
weights,
delimiter=',')
numpy.savetxt('biases_' + outputFnRoot + '.csv', biases, delimiter=',')
# calculate time taken for training
end_time = time.clock()
training_time = (end_time - start_time)
# save a list of the training parameters
if not os.path.exists('training_records.pkl'):
trainingRecords = []
else:
trainingRecords = cPickle.load(open('training_records.pkl', 'rb'))
record = {
'Corruption': corruptionLevel,
'# Feature': numFeatures,
'# Hidden Units': numHidden,
'# Epochs': epoch,
'Batch Size': batch_size,
'Initial Learning Rate': origLearningRate,
'Final Learning Rate': learning_rate,
'Delta Cost Stopping Threshold': deltaCostStopThreshold,
'Learning Rate Boost Factor': learningRateBoostFactor,
'Final Cost': lastEpochCost
}
trainingRecords.append(record)
cPickle.dump(trainingRecords, open('training_records.pkl', 'wb'))
# Print time taken for training
print >> sys.stderr, ('The ' + strCorruptionLevel +
'% corruption code for file ' +
os.path.split(__file__)[1] + ' ran for %.2fm' %
(training_time / 60.))
os.chdir('../')
def test_dA(dataset,
numFeatures,
numHidden,
learning_rate=0.1,
batch_size=20,
output_folder='dA_plots',
deltaCostStopThreshold=0.0005):
'''
Inputs:
:type learning_rate: float
:param learning_rate: learning rate used for training the DeNosing
AutoEncoder
:type training_epochs: int
:param training_epochs: number of epochs used for training
:type dataset: string
:param dataset: path to the picked dataset
'''
datasets = load_data(dataset)
train_set_x, train_set_y = datasets[0]
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size
# allocate symbolic variables for the data
index = T.lscalar() # index to a [mini]batch
x = T.matrix('x') # the data is presented as rasterized images
# Create output folder if it doesn't exist
if not os.path.isdir(output_folder):
os.makedirs(output_folder)
os.chdir(output_folder)
for corruptionLevel in numpy.arange(0.0, 1.01, 0.1):
strCorruptionLevel = str(int(100 * corruptionLevel))
######################
# BUILDING THE MODEL #
######################
rng = numpy.random.RandomState(123)
theano_rng = RandomStreams(rng.randint(2**30))
da = dA(numpy_rng=rng,
theano_rng=theano_rng,
input=x,
n_visible=numFeatures,
n_hidden=numHidden)
cost, updates = da.get_cost_updates(corruption_level=corruptionLevel,
learning_rate=learning_rate)
train_da = theano.function(
[index],
cost,
updates=updates,
givens={
x: train_set_x[index * batch_size:(index + 1) * batch_size]
})
start_time = time.clock()
############
# TRAINING #
############
# Go through training epochs until stopping criterion is met
deltaCost = 1
epoch = 0
lastEpochCost = 1e6
while abs(deltaCost) > deltaCostStopThreshold:
# Go through training set
c = []
for batch_index in xrange(n_train_batches):
c.append(train_da(batch_index))
epochCost = numpy.mean(c)
print 'Training epoch %d, cost ' % epoch, epochCost
deltaCost = epochCost - lastEpochCost
lastEpochCost = epochCost
epoch += 1
# Save final weights and biases
outputFnRoot = 'corruption_%s_nin_%s_nhdn_%s_basz_%s_lnrt_%s' \
% (strCorruptionLevel, str(numFeatures), str(numHidden), str(batch_size), str(learning_rate))
weights = da.W.get_value(borrow=True).T
biases = da.b.get_value(borrow=True)
numpy.savetxt('weights_' + outputFnRoot + '.csv',
weights,
delimiter=',')
numpy.savetxt('biases_' + outputFnRoot + '.csv', biases, delimiter=',')
# calculate time taken for training
end_time = time.clock()
training_time = (end_time - start_time)
# save a list of the training parameters
if not os.path.exists('training_records.pkl'):
trainingRecords = []
else:
trainingRecords = cPickle.load(open('training_records.pkl', 'rb'))
record = {
'Corruption': corruptionLevel,
'# Feature': numFeatures,
'# Hidden Units': numHidden,
'# Epochs': epoch,
'Batch Size': batch_size,
'Learning Rate': learning_rate,
'Delta Cost Stopping Threshold': deltaCostStopThreshold,
'Final Cost': epochCost
}
trainingRecords.append(record)
cPickle.dump(trainingRecords, open('training_records.pkl', 'wb'))
# Print time taken for training
print >> sys.stderr, ('The ' + strCorruptionLevel +
'% corruption code for file ' +
os.path.split(__file__)[1] + ' ran for %.2fm' %
(training_time / 60.))
os.chdir('../')
def test_dA(dataset,
numFeatures,
numHidden,
learning_rate=0.1,
training_epochs=15,
batch_size=20,
output_folder='dA_plots'):
'''
Inputs:
:type learning_rate: float
:param learning_rate: learning rate used for training the DeNosing
AutoEncoder
:type training_epochs: int
:param training_epochs: number of epochs used for training
:type dataset: string
:param dataset: path to the picked dataset
'''
datasets = load_data(dataset)
train_set_x, train_set_y = datasets[0]
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size
# allocate symbolic variables for the data
index = T.lscalar() # index to a [mini]batch
x = T.matrix('x') # the data is presented as rasterized images
# Create output folder if it doesn't exist
if not os.path.isdir(output_folder):
os.makedirs(output_folder)
os.chdir(output_folder)
for corruptionLevel in numpy.arange(0.0, 1.01, 0.1):
strCorruptionLevel = str(int(100 * corruptionLevel))
######################
# BUILDING THE MODEL #
######################
rng = numpy.random.RandomState(123)
theano_rng = RandomStreams(rng.randint(2**30))
da = dA(numpy_rng=rng,
theano_rng=theano_rng,
input=x,
n_visible=numFeatures,
n_hidden=numHidden)
cost, updates = da.get_cost_updates(corruption_level=corruptionLevel,
learning_rate=learning_rate)
train_da = theano.function(
[index],
cost,
updates=updates,
givens={
x: train_set_x[index * batch_size:(index + 1) * batch_size]
})
start_time = time.clock()
############
# TRAINING #
############
# go through training epochs
for epoch in xrange(training_epochs):
# go through training set
c = []
for batch_index in xrange(n_train_batches):
c.append(train_da(batch_index))
print 'Training epoch %d, cost ' % epoch, numpy.mean(c)
outputFnRoot = 'weights_corruption_%s_epoch_%s_nin_%s_nhdn_%s_basz_%s_lnrt_%s' \
% (strCorruptionLevel, str(epoch), str(numFeatures), str(numHidden), str(batch_size), str(learning_rate))
# save weights
weights = da.W.get_value(borrow=True).T
numpy.savetxt(outputFnRoot + '.csv', weights, delimiter=',')
# save image of weights
image = Image.fromarray(
tile_raster_images(X=weights,
img_shape=(1, numFeatures),
tile_shape=(numHidden, 1),
tile_spacing=(5, 5)))
image.save(outputFnRoot + '.png')
# calculate and print time taken for training
end_time = time.clock()
training_time = (end_time - start_time)
print >> sys.stderr, ('The ' + strCorruptionLevel +
'% corruption code for file ' +
os.path.split(__file__)[1] + ' ran for %.2fm' %
(training_time / 60.))
os.chdir('../')
help='The number of epochs',
type=int,
default=1)
parser.add_argument("-k",
'--clusters',
help='The number of clusters',
type=int,
default=2)
args = parser.parse_args()
if args.verbose > 0:
verbose = args.verbose
np.set_printoptions(suppress=True)
""" Solution Start """
print "Start"
data = load_data(vsData)
kmeans(data)
print "Hac min"
heights_min = hac(data)
print heights_min
heights_max = hac_max(data)
print heights_max
plt.figure()
h_min = hierarchy.linkage(heights_min, 'single')
hierarchy.dendrogram(h_min)
plt.savefig("./docs/single_hac.png")
plt.clf()