parser = argparse.ArgumentParser(
description='Twister2 Distributed MNIST Argument Parser')
parser.add_argument('--numberOfWorkers',
required=True,
help='number of workers')
parser.add_argument('--cpu', required=True, help='number of CPU per worker')
parser.add_argument('--ram', required=True, help='memory per worker')
parser.add_argument('--useGPU', required=True, help='train on GPU')
args = parser.parse_args()
# this is the entry point of the application
# specify how many workers you want
env = Twister2Environment(resources=[{
"cpu": int(args.cpu),
"ram": int(args.ram),
"instances": int(args.numberOfWorkers)
}])
worker_id = env.worker_id
numberOfWorkers = int(args.numberOfWorkers)
device = 'cpu' if args.useGPU == 'False' else 'gpu'
# TSet Source Function
# TSet is similar to Spark's RDD
class MNISTTrainingSource(SourceFunc):
def __init__(self, num_worker, worker_id):
super(MNISTTrainingSource, self).__init__()
import os
import math
import twister2deepnet.deepnet.util.utils as utils
# TWISTER2 DEEPNET IMPORTS
from twister2deepnet.deepnet.examples.MnistDistributed import MnistDistributed
from twister2deepnet.deepnet.io.FileUtils import FileUtils
DATA_FOLDER = '/tmp/twister2deepnet/mnist/'
TRAIN_DATA_SAVE_PATH = "/tmp/parquet/train/"
TEST_DATA_SAVE_PATH = "/tmp/parquet/test/"
PARALLELISM = 4
env = Twister2Environment(resources=[{
"cpu": 1,
"ram": 2048,
"instances": PARALLELISM
}],
mpi_aware=True)
comm = MPI.COMM_WORLD
world_size = PARALLELISM # int(os.environ['OMPI_COMM_WORLD_SIZE'])
world_rank = env.worker_id
TRAIN_DATA_FILE = str(world_rank) + ".data"
TRAIN_TARGET_FILE = str(world_rank) + ".target"
TEST_DATA_FILE = str(world_rank) + ".data"
TEST_TARGET_FILE = str(world_rank) + ".target"
TRAIN_DATA_FILES = [TRAIN_DATA_FILE, TRAIN_TARGET_FILE]
TEST_DATA_FILES = [TEST_DATA_FILE, TEST_TARGET_FILE]
from twister2.Twister2Environment import Twister2Environment
env = Twister2Environment(resources=[{"cpu": 1, "ram": 512, "instances": 4}])
# Your worker code starts here
print("Hello from worker %d" % env.worker_id)
import numpy as np
from twister2.Twister2Environment import Twister2Environment
from twister2.tset.fn.SourceFunc import SourceFunc
env = Twister2Environment(name="MyPython", config={"YOYO": 123}, resources=[{"cpu": 1, "ram": 1024, "instances": 2}])
class IntegerSource(SourceFunc):
def __init__(self):
super(IntegerSource, self).__init__()
self.x = 0
def has_next(self):
return self.x < 2
def next(self):
self.x += 1
return np.array([[1, 1, 1], [1, 1, 1.1]])
int_source = IntegerSource()
source = env.create_source(int_source, 4)
partitioned = source.partition(env.functions.partition.load_balanced)
def map(x):
x[1, 1] = 200
return x.reshape(6, 1)