def network_01(config): batch_size = config['batch_size'] sess = tf.Session() input_size = mnist.train.images.shape[1] x = tf.placeholder(tf.float32, [None, input_size]) y = tf.placeholder(tf.float32,[None,10]) autoencoder = create(x,y,config['autoencoder']) init = tf.initialize_all_variables() sess.run(init) dual_train_step = tf.train.GradientDescentOptimizer(0.5).minimize(autoencoder['cost_total']) class_train_step = tf.train.GradientDescentOptimizer(0.5).minimize(autoencoder['cost_class']) auto_train_step = tf.train.GradientDescentOptimizer(0.5).minimize(autoencoder['cost_autoencoder']) c1_axis = np.zeros(0) c2_axis = np.zeros(0) c3_axis = np.zeros(0) x_axis = np.zeros(0) if config['pre_training_batches'] > 0: """ PRETRAIN """ print 'pre-train autoencoder:' for i in tqdm(range(pre_training_batches)): batch_xs, batch_ys = mnist.train.next_batch(batch_size) sess.run(auto_train_step, feed_dict={x: batch_xs, y: batch_ys}) # do 1000 training stepscomp # print 'i\ttot\tclass\tauto' N = config['iterations'] print N for i in range(N): # Train classifier batch_xs, batch_ys = mnist.train.next_batch(config['batch_size']) if config['combined_cost_function']: sess.run(dual_train_step, feed_dict={x: batch_xs, y: batch_ys}) else: sess.run(class_train_step, feed_dict={x: batch_xs, y: batch_ys}) if config['alternating']: batch_xs, batch_ys = mnist.train.next_batch(batch_size) sess.run(auto_train_step, feed_dict={x: batch_xs, y: batch_ys}) if i % 100 == 0: batch_xs, batch_ys = mnist.validation.next_batch(batch_size) c1 = sess.run(autoencoder['cost_total'], feed_dict={x: batch_xs, y: batch_ys}) c2 = sess.run(autoencoder['cost_class'], feed_dict={x: batch_xs, y: batch_ys}) c3 = sess.run(autoencoder['cost_autoencoder'], feed_dict={x: batch_xs, y: batch_ys}) print i,c1,c2,c3 # x_axis = np.append(x_axis,i) # c1_axis = np.append(c1_axis,c1) # c2_axis = np.append(c2_axis,c2) # c3_axis = np.append(c3_axis,c3) sess.close()
train_couples_len = sum( 1 for e in utils.balance(couples_generator(images_train))) test_couples_len = sum(1 for e in utils.balance(couples_generator(images_test))) print('Training with %d couples.' % train_couples_len) print('Testing with %d couples.' % test_couples_len) print(input_shape) data_gen = utils.get_ImageDataGenerator(all_images, input_shape) train_iterator = utils.CouplesIterator(inf_couples_generator(images_train), input_shape, data_gen, BATCH_SIZE) test_iterator = utils.CouplesIterator(inf_couples_generator(images_test), input_shape, data_gen, BATCH_SIZE) model = network.create(input_shape) network.compile(model) model.save('pretrain.h5') model.fit_generator(train_iterator, steps_per_epoch=train_couples_len / BATCH_SIZE, epochs=EPOCHS) model.save('pretrain.h5') score = model.evaluate_generator(test_iterator, steps=test_couples_len / BATCH_SIZE) print(score) asd = utils.CouplesIterator(inf_couples_generator(images_test[:100]),
def cluster_creator(request, attributes, username): #Attributes passed in the form: # { # public_key: String, # workers: String, # password: String, # flavor: String, # status: Boolean # volSize: Integer - Not Required, # volume_name: String - Not Required # } #Store OpenRC Credentials for OpenStack connection credentials = get_credentials(attributes['tenant']) conn = openstack.connect(**credentials) #Store environment variables for use in new environment osdataproc_creds = env_credentials(attributes['tenant']) #Security Measure to ensure flavour exists in the requested tenant flavors=conn.list_flavors() flavor_names = [str(flavor.name) for flavor in flavors if flavor.id is not None] lustre_network = database.lustre_status(attributes['tenant']) with open('tenants_conf.yml', 'r') as file: data = yaml.load(file, Loader=yaml.Loader) lustre_image = data['lustre_image'].get('image') if attributes["flavor"] not in flavor_names: #If the flavor provided by the user is not a valid flavor #it'll be switched for m2.medium print("Invalid Flavor - Switching to default") attributes["flavor"] = "m2.medium" if attributes["status"] == False: if path.exists('/backend/clusters/'+username): #Implement better logging system for errors #This error indicates the cluster still exists despite the codebase believing it to be down print("A cluster is already registered - an error has occured!") else: #This runs if the tenant does not have secure lustre support #Creates the user's network network.create(conn, username, attributes['tenant']) #Run the cluster creation bash script if "volume_name" in attributes: #Volume Size is set to zero if volume exists for the user in the tenant volume_size = '0' print("Username: "******" is creating a cluster in " + attributes['tenant'] +", using volume: " + attributes['volume_name']) process = subprocess.run(['bash', 'cluster-creation.sh', username, attributes["password"], attributes["workers"], attributes["flavor"], attributes["volume_name"], volume_size, lustre_network, lustre_image], env = osdataproc_creds, capture_output=True, text=True) else: #If the user does not have a volume in the tenant, one is created for them #in the schema of USERNAME-cluster-volume volume_name = username+'-cluster-volume' database.add_volume(username, attributes['tenant'], volume_name) print("Creating Volume called: " + volume_name + " of size: " + attributes["volSize"]) print("Username: "******" is creating a cluster in " + attributes['tenant'] +", using volume: " + volume_name) process = subprocess.run(['bash', 'cluster-creation.sh', username, attributes["password"], attributes["workers"], attributes["flavor"], volume_name, attributes["volSize"], lustre_network, lustre_image], env = osdataproc_creds, capture_output=True, text=True) if DEBUG: print(process) try: path_to_cluster_ip = '/backend/clusters/' + username + '/osdataproc/terraform/terraform.tfstate.d/' + username + '/outputs.json' if path.exists(path_to_cluster_ip): with open(path_to_cluster_ip) as json_file: data = json.load(json_file) cluster_ip = data["spark_master_public_ip"]["value"] database.add_cluster(username, attributes['tenant'], cluster_ip, attributes['workers']) user_key_path = "/backend/clusters/"+username+"/pubkey.pub" cluster_location = "ubuntu@"+cluster_ip with open(user_key_path, 'w') as key_file: key_file.write(attributes["public_key"]) subprocess.run(['ssh-copy-id', '-f', '-i', user_key_path, cluster_location], capture_output=True, text=True) except: print("Error has occured when registering this cluster")
"Values": [instance_name] }]) terminated = False for reservation in res['Reservations']: for instance in reservation['Instances']: instance_state = instance['State']['Name'] print("------{}".format(instance_state)) if instance_state != 'terminated': terminated = True if terminated == False: break time.sleep(6) if __name__ == "__main__": res = network.create() create_pem() create_instance(res["subnet_id"], res["group_id"]) delete_instance() wait_instance_is_terminated() delete_pem() network.delete() ''' { "vpc_id":vpc_id, "gateway_id":gateway_id, "subnet_id":subnet_id, "group_id":group_id, "route_table_id":route_table_id } '''
def main(argv=None): # IGNORE:C0111 '''Command line options.''' from . import device if argv is None: argv = sys.argv else: sys.argv.extend(argv) program_name = os.path.basename(sys.argv[0]) program_version = "v%s" % __version__ program_build_date = str(__updated__) program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date) program_shortdesc = __shortdesc__ program_license = '''%s Created by dbsystem group on %s. Copyright 2016 NUS School of Computing. All rights reserved. Licensed under the Apache License 2.0 http://www.apache.org/licenses/LICENSE-2.0 Distributed on an "AS IS" basis without warranties or conditions of any kind, either express or implied. USAGE ''' % (program_shortdesc, str(__date__)) global debug try: # Setup argument parser parser = ArgumentParser(description=program_license, formatter_class=RawDescriptionHelpFormatter) parser.add_argument("-p", "--port", dest="port", default=5000, help="the port to listen to, default is 5000") parser.add_argument("-param", "--parameter", dest="parameter", help="the parameter file path to be loaded") parser.add_argument("-D", "--debug", dest="debug", action="store_true", help="whether need to debug") parser.add_argument("-R", "--reload", dest="reload_data", action="store_true", help="whether need to reload data") parser.add_argument("-C", "--cpu", dest="use_cpu", action="store_true", help="Using cpu or not, default is using gpu") parser.add_argument("-m", "--mode", dest="mode", choices=['train','test','serve'], default='serve', help="On Which mode (train,test,serve) to run singa") parser.add_argument('-V', '--version', action='version', version=program_version_message) # Process arguments args = parser.parse_args() port = args.port parameter_file = args.parameter mode = args.mode need_reload = args.reload_data use_cpu = args.use_cpu debug = args.debug #prepare data files config.read('file.cfg') file_prepare(need_reload) import network as net model = net.create() #load parameter parameter_file=get_parameter(parameter_file) if parameter_file: print "load parameter file: %s" % parameter_file model.load(parameter_file) if use_cpu: raise CLIError("Currently cpu is not support!") else: print "runing with gpu" d = device.create_cuda_gpu() model.to_device(d) if mode == "serve": print "runing singa in serve mode, listen to port: %s " % port global service from serve import Service service =Service(model,d) app.debug = debug app.run(host='0.0.0.0', port= port) elif mode == "train": print "runing singa in train mode" global trainer from train import Trainer trainer= Trainer(model,d) if not parameter_file: trainer.initialize() trainer.train() else: raise CLIError("Currently only serve mode is surpported!") return 0 except KeyboardInterrupt: ### handle keyboard interrupt ### return 0 except Exception, e: if debug: traceback.print_exc() raise(e) indent = len(program_name) * " " sys.stderr.write(program_name + ": " + str(e) + "\n") sys.stderr.write(indent + " for help use --help \n\n") return 2
def main(argv=None): # IGNORE:C0111 '''Command line options.''' from . import device if argv is None: argv = sys.argv else: sys.argv.extend(argv) program_name = os.path.basename(sys.argv[0]) program_version = "v%s" % __version__ program_build_date = str(__updated__) program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date) program_shortdesc = __shortdesc__ program_license = '''%s Created by dbsystem group on %s. Copyright 2016 NUS School of Computing. All rights reserved. Licensed under the Apache License 2.0 http://www.apache.org/licenses/LICENSE-2.0 Distributed on an "AS IS" basis without warranties or conditions of any kind, either express or implied. USAGE ''' % (program_shortdesc, str(__date__)) global debug try: # Setup argument parser parser = ArgumentParser(description=program_license, formatter_class=RawDescriptionHelpFormatter) parser.add_argument("-p", "--port", dest="port", default=5000, help="the port to listen to, default is 5000") parser.add_argument("-param", "--parameter", dest="parameter", help="the parameter file path to be loaded") parser.add_argument("-D", "--debug", dest="debug", action="store_true", help="whether need to debug") parser.add_argument("-R", "--reload", dest="reload_data", action="store_true", help="whether need to reload data") parser.add_argument("-C", "--cpu", dest="use_cpu", action="store_true", help="Using cpu or not, default is using gpu") parser.add_argument( "-m", "--mode", dest="mode", choices=['train', 'test', 'serve'], default='serve', help="On Which mode (train,test,serve) to run singa") parser.add_argument('-V', '--version', action='version', version=program_version_message) # Process arguments args = parser.parse_args() port = args.port parameter_file = args.parameter mode = args.mode need_reload = args.reload_data use_cpu = args.use_cpu debug = args.debug #prepare data files config.read('file.cfg') file_prepare(need_reload) import network as net model = net.create() #load parameter parameter_file = get_parameter(parameter_file) if parameter_file: print "load parameter file: %s" % parameter_file model.load(parameter_file) if use_cpu: raise CLIError("Currently cpu is not support!") else: print "runing with gpu" d = device.create_cuda_gpu() model.to_device(d) if mode == "serve": print "runing singa in serve mode, listen to port: %s " % port global service from serve import Service service = Service(model, d) app.debug = debug app.run(host='0.0.0.0', port=port) elif mode == "train": print "runing singa in train mode" global trainer from train import Trainer trainer = Trainer(model, d) if not parameter_file: trainer.initialize() trainer.train() else: raise CLIError("Currently only serve mode is surpported!") return 0 except KeyboardInterrupt: ### handle keyboard interrupt ### return 0 except Exception, e: if debug: traceback.print_exc() raise (e) indent = len(program_name) * " " sys.stderr.write(program_name + ": " + str(e) + "\n") sys.stderr.write(indent + " for help use --help \n\n") return 2