def worker(job_id): db = TensorDB(ip='146.169.33.34', port=27020, db_name='TransferGan', user_name='akara', password='******', studyID="MNIST") from bson.objectid import ObjectId job = db.find_one_job(args={'_id': ObjectId(job_id)}) if job['model'] == 'cnn': train_cnn(db=db, n_cnn_layers=job['n_cnn_layers'], lr=job['lr'], n_epochs=job['n_epochs']) elif job['model'] == 'mlp': train_mlp(db=db, n_layers=job['n_layers'], lr=job['lr'], n_epochs=job['n_epochs'])
def main(): db = TensorDB(ip='146.169.33.34', port=27020, db_name='TransferGan', user_name='akara', password='******', studyID="MNIST") create_mnist_dataset(db=db) create_jobs(db=db, job_name="cv_mnist", models_dict={ "cnn": { "lr": [0.01, 0.001, 0.001], "n_cnn_layers": [1, 2, 2], "n_filters": [64, 128, 256], "n_epochs": [10, 10, 10], }, "mlp": { "lr": [0.05, 0.0001], "n_layers": [1, 2], "n_epochs": [10, 10], } }) # Setting up the connection to interface ip = "interfaceeae.doc.ic.ac.uk" port = 443 eae = eAE(ip, port) # Testing if the interface is Alive is_alive = eae.is_eae_alive() if is_alive != 200: raise Exception("!!!") # Get all jobs jobs = db.get_all_jobs() args = [str(j['_id']) for j in jobs] # We submit a dummy job parameters_set = "\n".join(args) cluster = "gpu_dev" computation_type = "GPU" main_file = "/home/akara/Workspace/tl_paper/tutorial_tensordb_cv_mnist_worker.py" data_files = ['/home/akara/Workspace/tl_paper/tensorlayer'] host_ip = "dsihuaweiroom.doc.ic.ac.uk" ssh_port = "22" job = eae.submit_jobs(parameters_set, cluster, computation_type, main_file, data_files, host_ip, ssh_port) print(job)
def main(): # This is to initialize the connection to your MondonDB server # Note: make sure your MongoDB is reachable before changing this line db = TensorDB(ip='IP_ADDRESS_OR_YOUR_MONGODB', port=27017, db_name='DATABASE_NAME', user_name=None, password=None, studyID='ANY_ID (e.g., mnist)') create_mnist_dataset(db=db) create_jobs(db=db, job_name="cv_mnist", models_dict={ "cnn": { "lr": [0.01, 0.001, 0.001], "n_cnn_layers": [1, 2, 2], "n_filters": [64, 128, 256], "n_epochs": [10, 10, 10], }, "mlp": { "lr": [0.05, 0.0001], "n_layers": [1, 2], "n_epochs": [10, 10], } }) # Setting up the connection to interface ip = "IP_ADDRESS_OF_EAE (e.g., interfaceeae.doc.ic.ac.uk)" port = 443 eae = eAE(ip, port) # Testing if the interface is Alive is_alive = eae.is_eae_alive() if is_alive != 200: raise Exception("!!!") # Get all jobs jobs = db.get_all_jobs() args = [str(j['_id']) for j in jobs] # We submit a dummy job parameters_set = "\n".join(args) cluster = "NAME_OF_CLUSTER (e.g., gpu_dev)" computation_type = "COMPUTATION_TYPE (e.g., GPU)" main_file = "ABSOLUTE_PATH_TO_MAIN_FILE" data_files = ['ABSOLUTE_PATH_TO_DIRECTORY_OR_FILES_TO_BE_COPIED_TO_RUN_THE_MAIN_FILE'] host_ip = "IP_ADDRESS_OF_HOST_MACHINE_RUNNING_THIS_SCRIPT" ssh_port = "SSH_PORT_OF_HOST_MACHINE" job = eae.submit_jobs(parameters_set, cluster, computation_type, main_file, data_files, host_ip, ssh_port) print(job)
def worker(job_id): # This is to initialize the connection to your MondonDB server # Note: make sure your MongoDB is reachable before changing this line db = TensorDB(ip='IP_ADDRESS_OR_YOUR_MONGODB', port=27017, db_name='DATABASE_NAME', user_name=None, password=None, studyID='ANY_ID (e.g., mnist)') from bson.objectid import ObjectId job = db.find_one_job(args={'_id': ObjectId(job_id)}) if job['model'] == 'cnn': train_cnn(db=db, n_cnn_layers=job['n_cnn_layers'], lr=job['lr'], n_epochs=job['n_epochs']) elif job['model'] == 'mlp': train_mlp(db=db, n_layers=job['n_layers'], lr=job['lr'], n_epochs=job['n_epochs'])
def main(): # db = TensorDB(ip='146.169.33.34', port=27020, db_name='DRL', user_name='tensorlayer', password='******', studyID="20170524_1") db = TensorDB(ip='146.169.15.140', port=27017, db_name='DRL', user_name=None, password=None, studyID="1") # Create jobs n_jobs = 5 for j in range(n_jobs): args = { "id": j, "name": "Deep Reinforcement Learning", "file": "tutorial_tensordb_atari_pong_generator.py", "args": "", } db.submit_job(args=args) # Setting up the connection to interface ip = "interfaceeae.doc.ic.ac.uk" port = 443 eae = eAE(ip, port) # Testing if the interface is Alive is_alive = eae.is_eae_alive() if is_alive != 200: raise Exception("!!!") # Get all jobs jobs = db.get_jobs(status=JobStatus.WAITING) for j in jobs: # Start worker parameters_set = "--job_id={}".format(str(j["_id"])) cluster = "gpu" computation_type = "GPU" main_file = j["file"] data_files = ['tensorlayer'] host_ip = "dsigpu2.ict-doc.ic.ac.uk" ssh_port = "22222" job = eae.submit_jobs(parameters_set, cluster, computation_type, main_file, data_files, host_ip, ssh_port) db.change_job_status(job_id=j["_id"], status=JobStatus.RUNNING) print(job)
matplotlib.use("Agg") import matplotlib.pyplot as plt import numpy as np import tensorflow as tf from sklearn.metrics import confusion_matrix, f1_score from deepsleep.data_loader import NonSeqDataLoader, SeqDataLoader from deepsleep.model import DeepFeatureNet, DeepSleepNet from deepsleep.optimize import adam, adam_clipping_list_lr from deepsleep.utils import iterate_minibatches, iterate_batch_seq_minibatches from tensorlayer.db import TensorDB from tensorlayer.db import JobStatus db = TensorDB(ip='localhost', port=27017, db_name='DeepSleepNet', studyID='1') class Trainer(object): def __init__(self, interval_plot_filter=50, interval_save_model=15, interval_print_cm=10): self.interval_plot_filter = interval_plot_filter self.interval_save_model = interval_save_model self.interval_print_cm = interval_print_cm def print_performance(self, sess, output_dir, network_name, n_train_examples, n_valid_examples, train_cm, valid_cm, epoch, n_epochs, train_duration, train_loss, train_acc, train_f1, valid_duration,
from tensorlayer.db import TensorDB from tensorlayer.db import JobStatus db = TensorDB(ip='146.169.15.140', port=27017, db_name='DRL', user_name=None, password=None, studyID="1") # Terminate running jobs jobs = db.get_jobs(status=JobStatus.RUNNING) for j in jobs: print db.change_job_status(job_id=j["_id"], status=JobStatus.TERMINATED)
import gym import numpy as np import time, os import argparse from bson.objectid import ObjectId os.environ["CUDA_VISIBLE_DEVICES"] = "" # CPU from tensorlayer.db import TensorDB from tensorlayer.db import JobStatus # This is to initialize the connection to your MondonDB server # Note: make sure your MongoDB is reachable before changing this line db = TensorDB(ip='IP_ADDRESS_OR_YOUR_MONGODB', port=27017, db_name='DATABASE_NAME', user_name=None, password=None, studyID='ANY_ID (e.g., mnist)') def main(args): # hyperparameters image_size = 80 D = image_size * image_size H = 200 batch_size = 10 # learning_rate = 1e-4 gamma = 0.99 # decay_rate = 0.99 # render = False # display the game environment # resume = False # load existing policy network
image_size = 80 D = image_size * image_size H = 200 batch_size = 10 learning_rate = 1e-4 gamma = 0.99 decay_rate = 0.99 # render = False # display the game environment # resume = False # load existing policy network # model_file_name = "model_pong" np.set_printoptions(threshold=np.nan) from tensorlayer.db import TensorDB # db = TensorDB(ip='localhost', port=27017, db_name='atari', user_name=None, password=None) #<- if none password # db = TensorDB(ip='146.169.33.34', port=27020, db_name='DRL', user_name='tensorlayer', password='******', studyID='1') db = TensorDB(ip='146.169.15.140', port=27017, db_name='DRL', user_name=None, password=None, studyID='1') # def prepro(I): # """ prepro 210x160x3 uint8 frame into 6400 (80x80) 1D float vector """ # I = I[35:195] # I = I[::2,::2,0] # I[I == 144] = 0 # I[I == 109] = 0 # I[I != 0] = 1 # return I.astype(np.float).ravel() # env = gym.make("Pong-v0") # observation = env.reset() # prev_x = None # running_reward = None # reward_sum = 0
import matplotlib.pyplot as plt import numpy as np import tensorflow as tf from sklearn.metrics import confusion_matrix, f1_score from deepsleep.data_loader import NonSeqDataLoader, SeqDataLoader from deepsleep.model import DeepFeatureNet, DeepSleepNet from deepsleep.optimize import adam, adam_clipping_list_lr from deepsleep.utils import iterate_minibatches, iterate_batch_seq_minibatches from tensorlayer.db import TensorDB from tensorlayer.db import JobStatus db = TensorDB(ip='146.169.33.34', port=27020, db_name='DeepSleepNet', user_name='tensorlayer', password='******', studyID='1') class Trainer(object): def __init__(self, interval_plot_filter=50, interval_save_model=100, interval_print_cm=10): self.interval_plot_filter = interval_plot_filter self.interval_save_model = interval_save_model self.interval_print_cm = interval_print_cm def print_performance(self, sess, output_dir, network_name, n_train_examples, n_valid_examples, train_cm,