def __init__(self, model_keyword, batch_size, websocket_port):
Borg.__init__(self)
if 'model' not in self.__dict__:
print 'Creating new Borg worker'
if model_keyword == 'mnist':
self.model = mnistdnn.MnistDNN(batch_size, gpu=True)
elif model_keyword == 'higgs':
self.model = higgsdnn.HiggsDNN(batch_size)
elif model_keyword == 'molecular':
self.model = moleculardnn.MolecularDNN(batch_size)
else:
raise
self.batch_size = batch_size
self.websocket_port = websocket_port
self.loop = IOLoop.current()
self.loop.run_sync(self.init_websocket)
self.iteration = 0
#import sys
directory = "/user/root/"
model_keyword = 'higgs'
if model_keyword == 'mnist':
training_rdd_filename = '%smnist_train.csv' % directory
test_filename = '%smnist_test.csv' % directory
local_test_path = '/home/ubuntu/mnist_test.csv'
partitions = 48
warmup = 2000
batch_sz = 50
epochs = 5
repartition = True
time_lag = 100
model = mnistdnn.MnistDNN(batch_sz)
elif model_keyword == 'higgs':
training_rdd_filename = '%shiggs_train_all.csv' % directory
test_filename = '%shiggs_test_all.csv' % directory
local_test_path = '/home/ubuntu/higgs_test_all.csv'
warmup = 20000
epochs = 1
partitions = 64
batch_sz = 128
time_lag = 20
repartition = True
model = higgsdnn.HiggsDNN(batch_sz)
elif model_keyword == 'molecular':
training_rdd_filename = '%smolecular_train_all.csv' % directory
test_filename = '%smolecular_test_all.csv' % directory
local_test_path = '/home/ubuntu/molecular_test_all.csv'
import os
import random
MASTER_IP = '172.31.1.230'
SPARK_MASTER_PORT = 7077
SPARK_APP_NAME = 'Herp Derp'
HDFS_PORT = 9000
HDFS_DIRECTORY = '/mnist/'
LOCAL_DIRECTORY = "/home/ubuntu/ssp-ml/"
ERROR_RATES_PATH = "/home/ubuntu/ssp-ml/errors.txt"
WEBSOCKET_PORT = 8123 # random.randint(30000, 60000)
MODEL_KEYWORD = 'mnist'
if MODEL_KEYWORD == 'mnist':
TRAINING_RDD_FILENAME = ('hdfs://%s:%d' % (MASTER_IP, HDFS_PORT)) + \
os.path.join(HDFS_DIRECTORY, 'mnist_train.csv')
TEST_FILENAME = ('hdfs://%s:%d' % (MASTER_IP, HDFS_PORT)) + \
os.path.join(HDFS_DIRECTORY, 'mnist_test.csv')
LOCAL_TEST_PATH = os.path.join(LOCAL_DIRECTORY, 'mnist_test.csv')
NUM_PARTITIONS = 3
NUM_EPOCHS = 2
WARMUP = 2000
BATCH_SIZE = 50
#BATCH_SIZE = 0
EPOCHS = 5
REPARTITION = True
TIME_LAG = 100
MODEL = mnistdnn.MnistDNN(BATCH_SIZE)
else:
raise NotImplementedError('Currently only mnist model works')