def get_observers(args):
observers = []
if args['--mongo_db']:
url, db_name, prefix = _parse_mongo_db_arg(args['--mongo_db'])
if prefix:
mongo = MongoObserver.create(db_name=db_name, url=url,
prefix=prefix)
else:
mongo = MongoObserver.create(db_name=db_name, url=url)
observers.append(mongo)
return observers
def init():
log_id = str(int(time.time()*10)%(60*60*24*365*10))+str(os.getpid())
global logger
logger = logging.getLogger(str(log_id))
logger.setLevel(logging.DEBUG)
# write to file
fh = logging.FileHandler('ex.log')
fh.setLevel(logging.DEBUG)
# write to console
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
# Handler format
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s -\n\t%(message)s')
fh.setFormatter(formatter)
ch.setFormatter(formatter)
logger.addHandler(fh)
logger.addHandler(ch)
global ex
ex = Experiment('TrajMap')
ex.logger = logger
ex.observers.append(MongoObserver.create(url='10.60.43.110:27017', db_name='TurnHMM'))
#ex.observers.append(MongoObserver.create(url='127.0.0.1:27017', db_name='nTrajMap'))
return ex, logger
def objective(args_):
# arguments to pass as config_updates dict
global args
# result to pass to hyperopt
global result
# command-line arguments
global parse_args
try:
ex = Experiment('Hyperopt')
logger.debug("Adding observer for {}, DB {}".format(parse_args.mongo_db_address,parse_args.mongo_db_name))
ex.observers.append(MongoObserver.create(url=parse_args.mongo_db_address, db_name=parse_args.mongo_db_name))
pythia_args = make_args_for_pythia(args_)
args = mp.get_args(**pythia_args)
ex.main(run_with_global_args)
r = ex.run(config_updates=pythia_args)
logger.debug("Experiment result: {}\n"
"Report to hyperopt: {}".format(r.result, result))
return result
except:
raise
#If we somehow cannot get to the MongoDB server, then continue with the experiment
logger.warning("Running without Sacred")
run_with_global_args()
def set_up_xp():
# Check that MongoDB config is set
try:
mongo_uri=os.environ['PYTHIA_MONGO_DB_URI']
except KeyError as e:
print("Must define location of MongoDB in PYTHIA_MONGO_DB_URI for observer output",file=sys.stderr)
raise
ex = Experiment(ex_name)
ex.observers.append(MongoObserver.create(url=mongo_uri,
db_name=db_name))
return ex
def my_config():
learning_rate=0.13
n_epochs=1000,
data_path = "/home/drosen/repos/DeepLearningTutorials/data"
datasetname = 'mnist.small.pkl.gz'
#datasetname = 'mnist.pkl.gz'
dataset = os.path.join(data_path, datasetname)
batch_size=600
theano_flags = "mode=FAST_RUN,device=gpu,floatX=float32"
os.environ["THEANO_FLAGS"] = theano_flags
db_name = "MY_DB"
ex.observers.append(MongoObserver.create(db_name=db_name))
def my_config():
learning_rate = 0.01
n_epochs = 25
data_path = "/home/drosen/repos/DeepLearningTutorials/data"
datasetname = 'mnist.small.pkl.gz'
#datasetname = 'mnist.pkl.gz'
dataset = os.path.join(data_path, datasetname)
batch_size = 100
n_hidden = 500
theano_flags = "mode=FAST_RUN,device=gpu,floatX=float32"
os.environ["THEANO_FLAGS"] = theano_flags
db_name = "MY_DB"
ex.observers.append(MongoObserver.create(db_name=db_name))
random_seed = 1234
rng = numpy.random.RandomState(random_seed)
activation = "tanh"
def objective(args_):
global args
global result
global parse_args
args=args_
try:
ex = Experiment('Hyperopt')
ex.observers.append(MongoObserver.create(url=parse_args.mongo_db_address, db_name=parse_args.mongo_db_name))
ex.main(lambda: run_with_global_args())
r = ex.run(config_updates=args)
print(r)
return result
except:
raise
#If we somehow cannot get to the MongoDB server, then continue with the experiment
print("Running without Sacred")
run_with_global_args()
import os
import pandas as pd
import numpy as np
import datetime
from sklearn.ensemble import RandomForestClassifier as RF
from src.refined_rf import RefinedRandomForest as RRF
from sklearn.cross_validation import StratifiedKFold
from src.telstra_data import TelstraData, multiclass_log_loss
from sacred import Experiment
from sacred.observers import MongoObserver
ex = Experiment('refined_random_forest')
ex.observers.append(MongoObserver.create(db_name = "telstra"))
@ex.config
def my_config():
series = "RRF"
n_folds = 10
featureparams = {"location_min_count": 0,
"n_common_events":20,
"n_common_log_features":40,
"n_common_resources":5,
"n_label_encoded_events":4,
"n_label_encoded_log_features":4}
aggregateparams = {"loc_agg_prior_weight":3.0}
include = []
exclude = []
clfparams = {'n_estimators': 200,
'criterion': 'gini',
'max_depth': None,
'min_samples_split': 2,
from sacred.observers import MongoObserver
from dae import ex
nr_runs_per_dataset = 100
datasets = {
'bars': 12,
'corners': 5,
'shapes': 3,
'multi_mnist': 3,
'mnist_shape': 2,
'simple_superpos':2
}
db_name = 'binding_via_rc'
# Random search
ex.observers = [MongoObserver.create(db_name=db_name, prefix='random_search')]
for ds, k in datasets.items():
for i in range(nr_runs_per_dataset):
ex.run(config_updates={'dataset.name': ds, 'verbose': False, 'em.k': k},
named_configs=['random_search'])
# Multi-Train Runs
ex.observers = [MongoObserver.create(db_name=db_name, prefix='train_multi')]
for ds, k in datasets.items():
if ds == "simple_superpos": continue
for i in range(nr_runs_per_dataset):
ex.run(config_updates={
'dataset.name': ds,
'dataset.train_set': 'train_multi',
'em.k': k,
def apply(cls, args, run):
with open(args) as f:
auth = json.loads(f.read())
client = MongoClient(**auth['client_kwargs'])
mongo = MongoObserver.create(db_name=auth['db_name'], client=client)
run.observers.append(mongo)
def main():
import argparse
parser = argparse.ArgumentParser(description='Two layer linear regression')
parser.add_argument("image_feature_file_train",
type=str,
help="Image Feature file for the training set")
parser.add_argument("text_feature_file_train",
type=str,
help="Text Feature file for the training set")
parser.add_argument("image_feature_file_test",
type=str,
help="Image Feature file for the test set")
parser.add_argument("text_feature_file_test",
type=str,
help="Text Feature file for the test set")
parser.add_argument("word_vector_file",
type=str,
help="Text file containing the word vectors")
# Optional Args
parser.add_argument("--learning_rate",
type=float,
default=0.001,
help="Learning Rate")
parser.add_argument("--num_epochs",
type=int,
default=200,
help="Number of epochs to run for")
parser.add_argument("--batch_size",
type=int,
default=128,
help="Batch size to use for training")
parser.add_argument("--model_type",
type=str,
default="multihot",
choices=['multihot', 'naivesum', 'wdv', 'negsampling', 'fast0tag'],
help="Loss function to use for training")
parser.add_argument("--in_memory",
action='store_true',
default="store_false",
help="Load training image features into memory for faster training")
parser.add_argument("--model_input_path",
type=str,
default=None,
help="Model input path (to continue training)")
parser.add_argument("--model_output_path",
type=str,
default=None,
help="Model output path (to save training)")
# new args
parser.add_argument("--hidden_units",
type=str,
default="200",
help="Define a neural network as comma separated layer sizes. If log-reg, then set to '0'.")
parser.add_argument("--cross_eval",
action="store_true",
default=False,
help="Use if test dataset is different from training dataset")
parser.add_argument("--word_vector_type",
type=str,
choices=[t.name for t in WordVectorTypes],
help="Format of word_vector_file")
parser.add_argument("--epoch_verbosity",
type=int,
default=10,
help="Epoch verbosity rate")
parser.add_argument("--verbose_eval",
action="store_true",
default=False,
help="Use to run evaluation against test data every epoch_verbosity")
parser.add_argument("--optim_words",
action="store_true",
default=False,
help="If using negsampling model_type, use to jointly optimize words")
parser.add_argument("--ignore_posbatch",
action="store_true",
default=False,
help="Sample, ignoring from positive batch instead of examples. This should be taken out in future iters.")
parser.add_argument("--joint_factor",
type=float,
default=1.0,
help="Multiplier for learning rate in updating joint optimization")
parser.add_argument("--use_batch_norm",
action="store_true",
default=False,
help="Do we want to use batch normalization? Default is False")
parser.add_argument("--opt_type",
type=str,
default="adam",
help="What type of optimizer would you like? Choices are (adam,sgd)")
parser.add_argument("--weight_decay",
type=float,
default=0.0,
help="Weight decay to manually decay every 10 epochs. Default=0 for no decay.")
parser.add_argument("--scale_words",
type=float,
default=1.0,
help="Scale the word vectors. If set to zero, scale by L2-norm. Otherwise, wordvec=scale x wordvec")
parser.add_argument("--scale_images",
type=float,
default=1.0,
help="Scale the word vectors. If set to zero, scale by L2-norm. Otherwise, imvec=scale x imvec. ")
parser.add_argument("--fast_sample",
action="store_true",
default=False,
help="Fast sample based on distribution, only use in large dictionaries")
args = parser.parse_args()
try:
# Sacred Imports
from sacred import Experiment
from sacred.observers import MongoObserver
from sacred.initialize import Scaffold
# Monkey patch to avoid having to declare all our variables
def noop(item):
pass
Scaffold._warn_about_suspicious_changes = noop
ex = Experiment('Attalos')
ex.observers.append(MongoObserver.create(url=os.environ['MONGO_DB_URI'],
db_name='attalos_experiment'))
ex.main(lambda: convert_args_and_call_model(args))
ex.run(config_updates=args.__dict__)
except ImportError:
# We don't have sacred, just run the script
logger.warn('Not using Sacred. Your results will not be saved')
convert_args_and_call_model(args)
from config import SEMEVAL_HUMOR_TRAIN_DIR
from config import SEMEVAL_HUMOR_TRIAL_DIR
from config import TWEET_SIZE
from tf_tools import GPU_OPTIONS
from tf_tools import HUMOR_DROPOUT
from tf_tools import create_dense_layer
from tf_tools import create_tensorboard_visualization
from tf_tools import predict_on_hashtag
from tf_tools import build_humor_model
from tools import extract_tweet_pair_from_hashtag_datas
from tools import get_hashtag_file_names
from tools import load_hashtag_data
from tools import load_hashtag_data_and_vocabulary
ex = Experiment('humor_model')
ex.observers.append(MongoObserver.create(db_name='humor_runs'))
EMBEDDING_HUMOR_MODEL_LEARNING_RATE = .00001
N_TRAIN_EPOCHS = 2
@ex.config
def my_config():
learning_rate = .000005 # np.random.uniform(.00005, .0000005)
num_epochs = 5 # int(np.random.uniform(1.0, 4.0))
dropout = 1 # np.random.uniform(.5, 1.0)
hidden_dim_size = 800 # int(np.random.uniform(200, 3200))
use_emb_model = True
use_char_model = True
model_save_dir = EMB_CHAR_HUMOR_MODEL_DIR
if '-emb-only' in sys.argv:
def main():
import argparse
parser = argparse.ArgumentParser(description='Two layer linear regression')
parser.add_argument("image_feature_file_train",
type=str,
help="Image Feature file for the training set")
parser.add_argument("text_feature_file_train",
type=str,
help="Text Feature file for the training set")
parser.add_argument("image_feature_file_test",
type=str,
help="Image Feature file for the test set")
parser.add_argument("text_feature_file_test",
type=str,
help="Text Feature file for the test set")
parser.add_argument("word_vector_file",
type=str,
help="Text file containing the word vectors")
# Optional Args
parser.add_argument("--learning_rate",
type=float,
default=.001,
help="Learning Rate")
parser.add_argument("--epochs",
type=int,
default=200,
help="Number of epochs to run for")
parser.add_argument("--batch_size",
type=int,
default=128,
help="Batch size to use for training")
parser.add_argument("--network",
type=str,
default="200,200",
help="Define a neural network as comma separated layer sizes")
parser.add_argument("--model_type",
type=str,
default="mse",
choices=['mse', 'negsampling', 'fast0tag'],
help="Loss function to use for training")
parser.add_argument("--in_memory",
action='store_true',
default="store_false",
help="Load training image features into memory for faster training")
parser.add_argument("--model_input_path",
type=str,
default=None,
help="Model input path (to continue training)")
parser.add_argument("--model_output_path",
type=str,
default=None,
help="Model output path (to save training)")
parser.add_argument("--max_pos",
type=int,
default=5,
help="Max number of positive examples")
parser.add_argument("--max_neg",
type=int,
default=10,
help="Max number of negative examples")
global args
args = parser.parse_args()
try:
# Sacred Imports
from sacred import Experiment
from sacred.observers import MongoObserver
from sacred.initialize import Scaffold
# Monkey patch to avoid having to declare all our variables
def noop(item):
pass
Scaffold._warn_about_suspicious_changes = noop
ex = Experiment('Regress2sum')
ex.observers.append(MongoObserver.create(url=os.environ['MONGO_DB_URI'],
db_name='attalos_experiment'))
ex.main(lambda: convert_args_and_call_model())
ex.run(config_updates=args.__dict__)
except ImportError:
# We don't have sacred, just run the script
convert_args_and_call_model()
# write to console
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
# Handler format
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s -\n\t%(message)s')
fh.setFormatter(formatter)
ch.setFormatter(formatter)
logger.addHandler(fh)
logger.addHandler(ch)
ex = Experiment('Biagioni')
ex.logger = logger
ex.observers.append(MongoObserver.create(url='10.60.43.110:27017', db_name='Biagioni'))
@ex.config
def cfg():
#data_file ="../../Data/Shanghai/minsh_1000_biagioni"
data_file = "../Data/Chicago/all_trips"
@ex.automain
def main(data_file, side, k, percent, width, alpha, _log, _run):
_log.info('data_file: %s' % (data_file))
_run.info['time'] = {}
total_time = 0
