if args == 1: # starts master theads and spawns the relavent workers and creates communicators from the masters to workers comm_global = MPI.COMM_WORLD rank = comm_global.Get_rank() size = comm_global.Get_size() comm_children = MPI.COMM_WORLD.Spawn(sys.executable, args=[sys.argv[0], start_worker], maxprocs=n_workers * size) start = n_workers * rank end = start + n_workers - 1 if rank == 0: # on root starts a logger to save results f = l.DataLogger( "SPSA", n_layer, n_nodes, header= "Epoch,(cost,accuracy),Computation_Time,Train_Accuracy,Test_Accuracy", testing=True) else: # starts comminicator for worker comm_global = MPI.COMM_WORLD comm_parent = MPI.Comm.Get_parent() rank = comm_parent.Get_rank() size = comm_parent.Get_size() # Constructing the hidden and output layers def multilayer_perceptron(): prevlayer = x for w, b in zip(weights, biases):
from os import listdir from os.path import isfile, join import cPickle import numpy as np from mpi4py import MPI import time from sys import path from os import getcwd p = getcwd()[0:getcwd().rfind("/")]+"/SGD" path.append(p) p = getcwd()[0:getcwd().rfind("/")]+"/Logger" path.append(p) import Logger logfile = Logger.DataLogger("SVHN_LBFGS","Epoch,time,train_accuaracy,test_accuaracy,train_cost,test_cost") batch_size = 100 NUM_CLASSES = 10 from cifar10 import read_data_sets from ParamServer import ParamServer from ModelReplica import DPSGD comm = MPI.COMM_WORLD rank = comm.Get_rank() size = comm.Get_size() # Image processing for training the network. Note the many random # distortions applied to the image.
p = getcwd()[0:getcwd().rfind("/")] + "/MCMC" path.append(p) from Multi_try_Metropolis import MCMC from cifar10 import read_data_sets comm = MPI.COMM_WORLD rank = comm.Get_rank() size = comm.Get_size() NUM_CLASSES = 10 p = getcwd()[0:getcwd().rfind("/")] + "/Logger" path.append(p) import Logger logfile = Logger.DataLogger( "CIFAR10_MCMC", "Epoch,time,train_accuaracy,test_accuaracy,train_cost,test_cost") # Image processing for training the network. Note the many random # distortions applied to the image. # Randomly crop a [height, width] section of the image. #distorted_image = tf.random_crop(reshaped_image, [total_size, height, height,3]) # test_data = tf.random_crop(test_data, [total_size, height, height,3]) # # Randomly flip the image horizontally. # distorted_image = tf.image.random_flip_left_right(distorted_image) # # Because these operations are not commutative, consider randomizing # # the order their operation. # distorted_image = tf.image.random_brightness(distorted_image, # max_delta=63)
import time from sys import path from os import getcwd from tensorflow.examples.tutorials.mnist import input_data import tensorflow as tf from mpi4py import MPI p = getcwd()[0:getcwd().rfind("/")]+"/SGD" path.append(p) p = getcwd()[0:getcwd().rfind("/")]+"/Logger" path.append(p) import Logger logfile = Logger.DataLogger("MNIST_SGD","Epoch,time,train_accuaracy,test_accuaracy,train_cost,test_cost") from ParamServer import ParamServer from ModelReplica import DPSGD comm = MPI.COMM_WORLD rank = comm.Get_rank() size = comm.Get_size() mnist = input_data.read_data_sets("/tmp/data/", one_hot=True) # Parameters learning_rate = 0.001 training_epochs = 100 batch_size = 100