def network_01(config):
batch_size = config['batch_size']
sess = tf.Session()
input_size = mnist.train.images.shape[1]
x = tf.placeholder(tf.float32, [None, input_size])
y = tf.placeholder(tf.float32,[None,10])
autoencoder = create(x,y,config['autoencoder'])
init = tf.initialize_all_variables()
sess.run(init)
dual_train_step = tf.train.GradientDescentOptimizer(0.5).minimize(autoencoder['cost_total'])
class_train_step = tf.train.GradientDescentOptimizer(0.5).minimize(autoencoder['cost_class'])
auto_train_step = tf.train.GradientDescentOptimizer(0.5).minimize(autoencoder['cost_autoencoder'])
c1_axis = np.zeros(0)
c2_axis = np.zeros(0)
c3_axis = np.zeros(0)
x_axis = np.zeros(0)
if config['pre_training_batches'] > 0:
"""
PRETRAIN
"""
print 'pre-train autoencoder:'
for i in tqdm(range(pre_training_batches)):
batch_xs, batch_ys = mnist.train.next_batch(batch_size)
sess.run(auto_train_step, feed_dict={x: batch_xs, y: batch_ys})
# do 1000 training stepscomp
# print 'i\ttot\tclass\tauto'
N = config['iterations']
print N
for i in range(N):
# Train classifier
batch_xs, batch_ys = mnist.train.next_batch(config['batch_size'])
if config['combined_cost_function']:
sess.run(dual_train_step, feed_dict={x: batch_xs, y: batch_ys})
else:
sess.run(class_train_step, feed_dict={x: batch_xs, y: batch_ys})
if config['alternating']:
batch_xs, batch_ys = mnist.train.next_batch(batch_size)
sess.run(auto_train_step, feed_dict={x: batch_xs, y: batch_ys})
if i % 100 == 0:
batch_xs, batch_ys = mnist.validation.next_batch(batch_size)
c1 = sess.run(autoencoder['cost_total'], feed_dict={x: batch_xs, y: batch_ys})
c2 = sess.run(autoencoder['cost_class'], feed_dict={x: batch_xs, y: batch_ys})
c3 = sess.run(autoencoder['cost_autoencoder'], feed_dict={x: batch_xs, y: batch_ys})
print i,c1,c2,c3
# x_axis = np.append(x_axis,i)
# c1_axis = np.append(c1_axis,c1)
# c2_axis = np.append(c2_axis,c2)
# c3_axis = np.append(c3_axis,c3)
sess.close()
train_couples_len = sum(
1 for e in utils.balance(couples_generator(images_train)))
test_couples_len = sum(1
for e in utils.balance(couples_generator(images_test)))
print('Training with %d couples.' % train_couples_len)
print('Testing with %d couples.' % test_couples_len)
print(input_shape)
data_gen = utils.get_ImageDataGenerator(all_images, input_shape)
train_iterator = utils.CouplesIterator(inf_couples_generator(images_train),
input_shape, data_gen, BATCH_SIZE)
test_iterator = utils.CouplesIterator(inf_couples_generator(images_test),
input_shape, data_gen, BATCH_SIZE)
model = network.create(input_shape)
network.compile(model)
model.save('pretrain.h5')
model.fit_generator(train_iterator,
steps_per_epoch=train_couples_len / BATCH_SIZE,
epochs=EPOCHS)
model.save('pretrain.h5')
score = model.evaluate_generator(test_iterator,
steps=test_couples_len / BATCH_SIZE)
print(score)
asd = utils.CouplesIterator(inf_couples_generator(images_test[:100]),
def cluster_creator(request, attributes, username):
#Attributes passed in the form:
# {
# public_key: String,
# workers: String,
# password: String,
# flavor: String,
# status: Boolean
# volSize: Integer - Not Required,
# volume_name: String - Not Required
# }
#Store OpenRC Credentials for OpenStack connection
credentials = get_credentials(attributes['tenant'])
conn = openstack.connect(**credentials)
#Store environment variables for use in new environment
osdataproc_creds = env_credentials(attributes['tenant'])
#Security Measure to ensure flavour exists in the requested tenant
flavors=conn.list_flavors()
flavor_names = [str(flavor.name) for flavor in flavors if flavor.id is not None]
lustre_network = database.lustre_status(attributes['tenant'])
with open('tenants_conf.yml', 'r') as file:
data = yaml.load(file, Loader=yaml.Loader)
lustre_image = data['lustre_image'].get('image')
if attributes["flavor"] not in flavor_names:
#If the flavor provided by the user is not a valid flavor
#it'll be switched for m2.medium
print("Invalid Flavor - Switching to default")
attributes["flavor"] = "m2.medium"
if attributes["status"] == False:
if path.exists('/backend/clusters/'+username):
#Implement better logging system for errors
#This error indicates the cluster still exists despite the codebase believing it to be down
print("A cluster is already registered - an error has occured!")
else:
#This runs if the tenant does not have secure lustre support
#Creates the user's network
network.create(conn, username, attributes['tenant'])
#Run the cluster creation bash script
if "volume_name" in attributes:
#Volume Size is set to zero if volume exists for the user in the tenant
volume_size = '0'
print("Username: "******" is creating a cluster in " + attributes['tenant'] +", using volume: " + attributes['volume_name'])
process = subprocess.run(['bash', 'cluster-creation.sh', username, attributes["password"],
attributes["workers"], attributes["flavor"], attributes["volume_name"],
volume_size, lustre_network, lustre_image], env = osdataproc_creds, capture_output=True, text=True)
else:
#If the user does not have a volume in the tenant, one is created for them
#in the schema of USERNAME-cluster-volume
volume_name = username+'-cluster-volume'
database.add_volume(username, attributes['tenant'], volume_name)
print("Creating Volume called: " + volume_name + " of size: " + attributes["volSize"])
print("Username: "******" is creating a cluster in " + attributes['tenant'] +", using volume: " + volume_name)
process = subprocess.run(['bash', 'cluster-creation.sh', username, attributes["password"],
attributes["workers"], attributes["flavor"], volume_name, attributes["volSize"], lustre_network, lustre_image],
env = osdataproc_creds, capture_output=True, text=True)
if DEBUG:
print(process)
try:
path_to_cluster_ip = '/backend/clusters/' + username + '/osdataproc/terraform/terraform.tfstate.d/' + username + '/outputs.json'
if path.exists(path_to_cluster_ip):
with open(path_to_cluster_ip) as json_file:
data = json.load(json_file)
cluster_ip = data["spark_master_public_ip"]["value"]
database.add_cluster(username, attributes['tenant'], cluster_ip, attributes['workers'])
user_key_path = "/backend/clusters/"+username+"/pubkey.pub"
cluster_location = "ubuntu@"+cluster_ip
with open(user_key_path, 'w') as key_file:
key_file.write(attributes["public_key"])
subprocess.run(['ssh-copy-id', '-f', '-i', user_key_path, cluster_location], capture_output=True, text=True)
except:
print("Error has occured when registering this cluster")
"Values": [instance_name]
}])
terminated = False
for reservation in res['Reservations']:
for instance in reservation['Instances']:
instance_state = instance['State']['Name']
print("------{}".format(instance_state))
if instance_state != 'terminated':
terminated = True
if terminated == False:
break
time.sleep(6)
if __name__ == "__main__":
res = network.create()
create_pem()
create_instance(res["subnet_id"], res["group_id"])
delete_instance()
wait_instance_is_terminated()
delete_pem()
network.delete()
'''
{
"vpc_id":vpc_id,
"gateway_id":gateway_id,
"subnet_id":subnet_id,
"group_id":group_id,
"route_table_id":route_table_id
}
'''
def main(argv=None): # IGNORE:C0111
'''Command line options.'''
from . import device
if argv is None:
argv = sys.argv
else:
sys.argv.extend(argv)
program_name = os.path.basename(sys.argv[0])
program_version = "v%s" % __version__
program_build_date = str(__updated__)
program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date)
program_shortdesc = __shortdesc__
program_license = '''%s
Created by dbsystem group on %s.
Copyright 2016 NUS School of Computing. All rights reserved.
Licensed under the Apache License 2.0
http://www.apache.org/licenses/LICENSE-2.0
Distributed on an "AS IS" basis without warranties
or conditions of any kind, either express or implied.
USAGE
''' % (program_shortdesc, str(__date__))
global debug
try:
# Setup argument parser
parser = ArgumentParser(description=program_license, formatter_class=RawDescriptionHelpFormatter)
parser.add_argument("-p", "--port", dest="port", default=5000, help="the port to listen to, default is 5000")
parser.add_argument("-param", "--parameter", dest="parameter", help="the parameter file path to be loaded")
parser.add_argument("-D", "--debug", dest="debug", action="store_true", help="whether need to debug")
parser.add_argument("-R", "--reload", dest="reload_data", action="store_true", help="whether need to reload data")
parser.add_argument("-C", "--cpu", dest="use_cpu", action="store_true", help="Using cpu or not, default is using gpu")
parser.add_argument("-m", "--mode", dest="mode", choices=['train','test','serve'], default='serve', help="On Which mode (train,test,serve) to run singa")
parser.add_argument('-V', '--version', action='version', version=program_version_message)
# Process arguments
args = parser.parse_args()
port = args.port
parameter_file = args.parameter
mode = args.mode
need_reload = args.reload_data
use_cpu = args.use_cpu
debug = args.debug
#prepare data files
config.read('file.cfg')
file_prepare(need_reload)
import network as net
model = net.create()
#load parameter
parameter_file=get_parameter(parameter_file)
if parameter_file:
print "load parameter file: %s" % parameter_file
model.load(parameter_file)
if use_cpu:
raise CLIError("Currently cpu is not support!")
else:
print "runing with gpu"
d = device.create_cuda_gpu()
model.to_device(d)
if mode == "serve":
print "runing singa in serve mode, listen to port: %s " % port
global service
from serve import Service
service =Service(model,d)
app.debug = debug
app.run(host='0.0.0.0', port= port)
elif mode == "train":
print "runing singa in train mode"
global trainer
from train import Trainer
trainer= Trainer(model,d)
if not parameter_file:
trainer.initialize()
trainer.train()
else:
raise CLIError("Currently only serve mode is surpported!")
return 0
except KeyboardInterrupt:
### handle keyboard interrupt ###
return 0
except Exception, e:
if debug:
traceback.print_exc()
raise(e)
indent = len(program_name) * " "
sys.stderr.write(program_name + ": " + str(e) + "\n")
sys.stderr.write(indent + " for help use --help \n\n")
return 2
def main(argv=None): # IGNORE:C0111
'''Command line options.'''
from . import device
if argv is None:
argv = sys.argv
else:
sys.argv.extend(argv)
program_name = os.path.basename(sys.argv[0])
program_version = "v%s" % __version__
program_build_date = str(__updated__)
program_version_message = '%%(prog)s %s (%s)' % (program_version,
program_build_date)
program_shortdesc = __shortdesc__
program_license = '''%s
Created by dbsystem group on %s.
Copyright 2016 NUS School of Computing. All rights reserved.
Licensed under the Apache License 2.0
http://www.apache.org/licenses/LICENSE-2.0
Distributed on an "AS IS" basis without warranties
or conditions of any kind, either express or implied.
USAGE
''' % (program_shortdesc, str(__date__))
global debug
try:
# Setup argument parser
parser = ArgumentParser(description=program_license,
formatter_class=RawDescriptionHelpFormatter)
parser.add_argument("-p",
"--port",
dest="port",
default=5000,
help="the port to listen to, default is 5000")
parser.add_argument("-param",
"--parameter",
dest="parameter",
help="the parameter file path to be loaded")
parser.add_argument("-D",
"--debug",
dest="debug",
action="store_true",
help="whether need to debug")
parser.add_argument("-R",
"--reload",
dest="reload_data",
action="store_true",
help="whether need to reload data")
parser.add_argument("-C",
"--cpu",
dest="use_cpu",
action="store_true",
help="Using cpu or not, default is using gpu")
parser.add_argument(
"-m",
"--mode",
dest="mode",
choices=['train', 'test', 'serve'],
default='serve',
help="On Which mode (train,test,serve) to run singa")
parser.add_argument('-V',
'--version',
action='version',
version=program_version_message)
# Process arguments
args = parser.parse_args()
port = args.port
parameter_file = args.parameter
mode = args.mode
need_reload = args.reload_data
use_cpu = args.use_cpu
debug = args.debug
#prepare data files
config.read('file.cfg')
file_prepare(need_reload)
import network as net
model = net.create()
#load parameter
parameter_file = get_parameter(parameter_file)
if parameter_file:
print "load parameter file: %s" % parameter_file
model.load(parameter_file)
if use_cpu:
raise CLIError("Currently cpu is not support!")
else:
print "runing with gpu"
d = device.create_cuda_gpu()
model.to_device(d)
if mode == "serve":
print "runing singa in serve mode, listen to port: %s " % port
global service
from serve import Service
service = Service(model, d)
app.debug = debug
app.run(host='0.0.0.0', port=port)
elif mode == "train":
print "runing singa in train mode"
global trainer
from train import Trainer
trainer = Trainer(model, d)
if not parameter_file:
trainer.initialize()
trainer.train()
else:
raise CLIError("Currently only serve mode is surpported!")
return 0
except KeyboardInterrupt:
### handle keyboard interrupt ###
return 0
except Exception, e:
if debug:
traceback.print_exc()
raise (e)
indent = len(program_name) * " "
sys.stderr.write(program_name + ": " + str(e) + "\n")
sys.stderr.write(indent + " for help use --help \n\n")
return 2