def test_getarray_noinit_noinit():
"""Test that calling on a non-existant array does not initialize that array"""
arr = np.ones((5, 5))
model = BasicModel(data=arr)
try:
model.getarray_noinit('area')
except AttributeError:
pass
assert 'area' not in model.instance
def test_skip_serializing_null(tmp_path, filename):
"""Make sure that None is not written out to the ASDF tree"""
file_path = tmp_path / filename
with BasicModel() as model:
model.meta.telescope = None
model.save(file_path)
with BasicModel(file_path) as model:
# Make sure that 'telescope' is not in the tree
with pytest.raises(KeyError):
assert model.meta["telescope"] is None
def test_stringify(tmp_path):
dm = DataModel()
assert str(dm) == '<DataModel>'
dm = BasicModel((10, 100))
assert str(dm) == '<BasicModel(10, 100)>'
file_path = tmp_path / "test.asdf"
dm.save(file_path)
dm.close()
with BasicModel(file_path) as dm:
assert str(dm) == '<BasicModel(10, 100) from test.asdf>'
def test_secondary_shapes():
"""
Confirm that a non-primary array takes on the shape
specified in the initializer.
"""
with BasicModel((256, 256)) as dm:
assert dm.area.shape == (256, 256)
def get_default_config(self):
config = BasicModel.get_default_config(self)
model_config = {
'stride':
1,
'inception_v4_checkpoint_file':
os.path.join(script_dir, '..', 'data', 'inception_v4.ckpt'),
'batch_norm_decay':
0.99,
'batch_norm_epsilon':
0.001,
'output_size':
29,
'pad':
32,
'receptive_field_size':
66,
'projective_field_size':
7,
'contextual_pad':
32,
'normalize_inputs':
False,
'batch_size':
64,
}
config.update(model_config)
return config
def test_object_node_iterator():
m = BasicModel({"meta": {"foo": "bar"}})
items = []
for i in m.meta.items():
items.append(i[0])
assert 'foo' in items
def test_broadcast2():
with BasicModel() as dm:
data = np.empty((52, 50))
dm.data = data
dq = np.empty((50, ))
dm.dq = dq
def test_update_from_dict(tmp_path):
"""Test update method from a dictionary"""
file_path = tmp_path / "update.asdf"
with BasicModel((5, 5)) as m:
m.update({"foo": "bar", "baz": 42})
m.save(file_path)
with asdf.open(file_path) as af:
assert af["foo"] == "bar"
assert af["baz"] == 42
def get_default_config(self):
config = BasicModel.get_default_config(self)
model_config = {
'stride': 1,
'img_rows_target': 289,
'img_cols_target': 289,
'normalize_inputs': True,
}
config.update(model_config)
return config
def test_init_from_pathlib(tmp_path):
"""Test initializing model from a PurePath object"""
file_path = tmp_path / "test.asdf"
with asdf.AsdfFile() as af:
af["meta"] = {"telescope": "crystal ball"}
af.write_to(file_path)
model = BasicModel(file_path)
# Test is basically, did we open the model?
assert model.meta.telescope == "crystal ball"
def evaluate_line():
with open(FLAGS.map_file, 'rb') as f:
char_to_id, id_to_char, tag_to_id, id_to_tag = cPickle.load(f)
config=BasicModelConfig(FLAGS,len(char_to_id),len(tag_to_id),4)
tfConfig = tf.ConfigProto()
tfConfig.gpu_options.per_process_gpu_memory_fraction = FLAGS.memory_usage
with tf.Session(config=tfConfig) as sess:
model = BasicModel(config, None)
saver = tf.train.Saver()
model_name = os.path.join(FLAGS.checkpoints, FLAGS.model_name)
saver.restore(sess, model_name)
while True:
line = raw_input('\n请输入测试句子(0:Exit):\n'.encode('utf-8')).decode('utf-8')
if line=='0':
exit()
strings, predicts, _ = model.evaluate_step(sess, input_from_line(line, char_to_id))
string=strings[0]
tags=[id_to_tag[t] for t in predicts[0]]
assert len(string)==len(tags)
result_json=result_to_json(string,tags)
print_result_json(result_json)
def test_copy():
with BasicModel((50, 50)) as dm:
dm.meta.telescope = "MEADE"
dm.meta.foo = "BAR"
with dm.copy() as dm2:
dm2.data[0, 0] = 42
assert np.sum(dm.data.flatten()) == 0
assert dm2.meta.telescope == "MEADE"
assert dm2.meta.foo == "BAR"
dm2.meta.foo = "BAZ"
assert dm.meta.foo == "BAR"
dm2.meta.origin = "STScI"
assert dm.meta.origin is None
def get_default_config(self):
config = BasicModel.get_default_config(self)
model_config = {
'stride':
1,
'inception_v4_checkpoint_file':
os.path.join(script_dir, '..', 'data', 'inception_v4.ckpt'),
'batch_norm_decay':
0.9,
'batch_norm_epsilon':
0.001,
'output_size':
29, # Projective field size at Mixed_5b
'pad':
32,
'receptive_field_size':
2 * 33,
'projective_field_size':
7,
'target_context_pad':
23, # ~(7./2.) / (128./33.)
'target_embedding_pad':
16,
# The contextual pad should be less than 48, which is half of the
# difference of the receptive fields of Mixed_5a and Mixed_5d, i.e. the
# number of pixels that the additional receptive field can look ahead. It
# should also not include so much black borders from data augmentation.
'contextual_pad':
32,
# Excluding the additional pad needed in order to get the full receptive
# field into view.
'large_contextual_pad_unpadded':
128,
'large_contextual_pad':
128 + 66,
'normalize_inputs':
False,
'batch_size':
4,
'hidden_state_size':
96,
'draw_border':
True,
'loss_function':
'l17',
}
config.update(model_config)
return config
def test_implicit_creation_lower_dimensionality():
with BasicModel(np.zeros((10, 20))) as m:
assert m.dq.shape == (20, )
e2_classes=E2_CLASSES,
model_id=model_id,
input_dim=n_samples,
out_dim=model_out_dim,
)
elif isMode(mode, 'e1'):
model = E1Model(
classes=E1_CLASSES,
model_id=model_id,
input_dim=n_samples,
out_dim=model_out_dim,
)
elif isMode(mode, 'r'):
model = BasicModel(
input_dim=n_samples,
out_dim=model_out_dim,
model_id=model_id,
)
print("Model {}".format(model_id))
# Prepare Data for prediction
if isMode(mode, 'e1_e2'):
pc = PermittivityCalculator()
e1_classes, e2_classes = list(set(r_e1_e2_data['e1_cls'])), list(
set(r_e1_e2_data['e2_cls']))
e1e2_prediction_classes = [
f'{e1_cls},{e2_cls}' for e1_cls in e1_classes for e2_cls in e2_classes
]
def test_initialize_arrays_with_arglist():
shape = (10, 10)
area = np.full((2, 2), 13.0)
m = BasicModel(shape, area=area)
assert np.array_equal(m.area, area)
def train():
# load data sets
train_sentences=load_sentences(FLAGS.train_file,FLAGS.zeros)
dev_sentences=load_sentences(FLAGS.dev_file,FLAGS.zeros)
test_sentences=load_sentences(FLAGS.test_file,FLAGS.zeros)
# appoint tagging scheme (IOB/IOBES)
train_sentences=update_tag_scheme(train_sentences,FLAGS.tag_schema)
dev_sentences=update_tag_scheme(dev_sentences,FLAGS.tag_schema)
test_sentences=update_tag_scheme(test_sentences,FLAGS.tag_schema)
#create maps if not exist
if not os.path.exists(FLAGS.map_file):
if FLAGS.pre_emb:
char_to_id,_=char_mapping(train_sentences)
char_to_id,id_to_char=augment_with_pretrained(char_to_id,'wiki_100.utf8')
else:
char_to_id, id_to_char=char_mapping(train_sentences)
tag_to_id, id_to_tag=tag_mapping(train_sentences)
with open(FLAGS.map_file,'wb') as f:
cPickle.dump([char_to_id,id_to_char,tag_to_id,id_to_tag],f,cPickle.HIGHEST_PROTOCOL)
else:
with open(FLAGS.map_file,'rb') as f:
char_to_id, id_to_char, tag_to_id, id_to_tag=cPickle.load(f)
# prepare data, get a collection of list containing index
train_data=prepare_dataset(train_sentences,char_to_id,tag_to_id,True)
dev_data=prepare_dataset(dev_sentences,char_to_id,tag_to_id,True)
test_data=prepare_dataset(test_sentences,char_to_id,tag_to_id,True)
print "%i %i %i sentences in train / dev / test." % (len(train_data),len(dev_data),len(test_data))
if not FLAGS.pre_emb:
pre_emb=None
else:
pre_emb=load_word2vec(FLAGS.pre_emb_file,char_to_id,FLAGS.char_dim)
print "init embedding shape: (%d,%d)" %(pre_emb.shape[0],pre_emb.shape[1])
train_manager=BatchManager(train_data,FLAGS.batch_size,True)
dev_manager=BatchManager(dev_data,FLAGS.batch_size,False)
test_manager=BatchManager(test_data,FLAGS.batch_size,False)
config=BasicModelConfig(FLAGS,len(char_to_id),len(tag_to_id),4)
tfConfig = tf.ConfigProto()
tfConfig.gpu_options.per_process_gpu_memory_fraction = FLAGS.memory_usage
with tf.Session(config=tfConfig) as sess:
print "Train started!"
model=BasicModel(config,pre_emb)
saver=tf.train.Saver()
# tensorboard
if not os.path.exists(FLAGS.summaries_dir):
os.mkdir(FLAGS.summaries_dir)
merged=tf.summary.merge_all()
train_writer=tf.summary.FileWriter(os.path.join(FLAGS.summaries_dir,FLAGS.model_name,"train"),sess.graph)
test_writer=tf.summary.FileWriter(os.path.join(FLAGS.summaries_dir,FLAGS.model_name,"test"),sess.graph)
# load previous trained model or create a new model
if not os.path.exists(FLAGS.checkpoints):
os.mkdir(FLAGS.checkpoints)
model_name=os.path.join(FLAGS.checkpoints,FLAGS.model_name)
ckpt=tf.train.get_checkpoint_state(FLAGS.checkpoints)
if ckpt and ckpt.model_checkpoint_path:
print "restore from previous traied model: %s" % FLAGS.model_name
saver.restore(sess,ckpt.model_checkpoint_path)
else:
sess.run(tf.global_variables_initializer())
def evaluate(sess,model,manager):
strings=[]
predicts=[]
goldens=[]
bar = ProgressBar(max_value=manager.num_batch)
for batch in bar(manager.iter_batch()):
batch_string,batch_predict,batch_golden=model.evaluate_step(sess,batch)
strings.extend(batch_string)
predicts.extend(batch_predict)
goldens.extend(batch_golden)
return strings,predicts,goldens
best_eval_f1=0
noimpro_num=0
for i in range(FLAGS.max_epoch):
#train
train_loss=[]
bar = ProgressBar(max_value=train_manager.num_batch)
for step,batch in bar(enumerate(train_manager.iter_batch())):
batch.append(merged)
summary,global_step,batch_loss=model.train_step(sess,batch,FLAGS.dropout_keep)
#add summary to tensorboard
train_writer.add_summary(summary,global_step)
train_loss.append(batch_loss)
print "Epoch %d Train loss is %.4f" % (i+1,np.mean(train_loss))
#dev
strings,predicts,goldens=evaluate(sess,model,dev_manager)
eval_f1=report_results(strings,predicts,goldens,id_to_char,id_to_tag,'outputs/dev')
if eval_f1>best_eval_f1:
best_eval_f1=eval_f1
noimpro_num=0
saver.save(sess,model_name)
else:
noimpro_num+=1
print "Epoch %d Best eval f1:%.6f" % (i+1,best_eval_f1)
#test
strings,predicts,goldens=evaluate(sess,model,test_manager)
test_f1=report_results(strings,predicts,goldens,id_to_char,id_to_tag,'outputs/test',True)
#early_stop
if noimpro_num>=3:
print "Early stop! Final F1 scores on test data is :%.6f" % test_f1
break
print
def test_getarray_noinit_valid():
"""Test for valid value return"""
arr = np.ones((5, 5))
model = BasicModel(data=arr)
fetched = model.getarray_noinit('data')
assert (fetched == arr).all()
def test_getarray_noinit_raises():
"""Test for error when accessing non-existent array"""
arr = np.ones((5, 5))
model = BasicModel(data=arr)
with pytest.raises(AttributeError):
model.getarray_noinit('area')
def test_init_with_array2():
with pytest.raises(ValueError):
array = np.empty((50, ))
with BasicModel(array) as dm:
dm.data
def test_set_array():
with pytest.raises(ValueError):
with BasicModel() as dm:
data = np.empty((50, ))
dm.data = data
def test_set_shape():
with BasicModel((50, 50)) as dm:
assert dm.shape == (50, 50)
with pytest.raises(AttributeError):
dm.shape = (42, 23)
def test_broadcast():
with BasicModel((50, 50)) as dm:
data = np.empty((50, ))
dm.dq = data
dynamic_utility_parameter_distribution = True
else:
dynamic_utility_parameter_distribution = False
sampling_policy_name = 'uEI_prior'
elif sampling_policy_name is 'uTS':
model = MultiOutputGP(output_dim=m,
exact_feval=[True] * m,
fixed_hyps=False) # Model (Multi-output GP)
sampling_policy = uTS(model, space, optimizer='CMA', utility=utility)
if learn_preferences:
dynamic_utility_parameter_distribution = True
else:
dynamic_utility_parameter_distribution = False
sampling_policy_name = 'uTS_prior'
elif sampling_policy_name is 'Random':
model = BasicModel(output_dim=m)
sampling_policy = Random(model=None, space=space)
dynamic_utility_parameter_distribution = False
elif sampling_policy_name is 'ParEGO':
model = BasicModel(output_dim=m)
sampling_policy = ParEGO(model, space, utility)
dynamic_utility_parameter_distribution = False
# BO model
max_iter = 150
experiment_name = 'test_dtlz1a'
if len(sys.argv) > 1:
experiment_number = int(sys.argv[1])
filename = [
experiment_name, sampling_policy_name,
str(experiment_number)
def test_search_schema():
with BasicModel() as x:
results = x.search_schema('origin')
assert [x[0] for x in results] == ['meta.origin']
def test_delete():
with BasicModel() as dm:
dm.meta.telescope = 'JWST'
assert dm.meta.telescope == 'JWST'
del dm.meta.telescope
assert dm.meta.telescope is None
def main(args, stream):
if not args.class_wise_sampling and args.data_seed is None:
args.data_seed = random.randint(1, 1e8)
# * Prepare data_module *
dm = DInterface(**vars(args))
args.class_dict = dm.init_data['class_dict']
args.classes = list(args.class_dict.keys())
args.num_classes = len(args.class_dict)
global_bs = args.gpus * args.batch_size if args.gpus > 1 else args.batch_size
# * Build model *
net = build_model(**vars(args))
if args.load_pretrained:
pretrained_path = load_pretrain_path_by_args(args, '.pth.tar')
bl_layers = None
if args.mode_name in ['train', 'finetune']:
bl_layers = ['classifier', 'fc']
net = load_ckpt(net, pretrained_path,
train=(args.mode_name == 'train'),
block_layers=bl_layers,
map_keys=args.map_keys,
verbose=True)
model = BasicModel(net, **vars(args))
# Resume
load_path = load_model_path_by_args(args)
if load_path is not None:
model.load_from_checkpoint(checkpoint_path=load_path, strict=False)
# * validate mode *
if args.mode_name in ['val', 'test']:
model.final_val = True
trainer = Trainer.from_argparse_args(args)
trainer.validate(model, datamodule=dm)
return
# * Callbacks *
# Checkpoint callbacks
if args.ckpt == 'debug' or not args.save_ckpt:
# ckpt_callback = get_checkpoint_callback(args.ckpt, save_last=False, save_top_k=0)
ckpt_callback = get_checkpoint_callback(f'Task_models/{args.net_suffix}', save_last=False, save_top_k=1)
else:
cpDir = '{}/{}_{}'.format(args.ckpt, args.model_name, args.net_suffix)
every_n_train_steps = dm.num_samples//global_bs
if args.ckpt_ever_n_epoch:
every_n_train_steps *= args.ckpt_ever_n_epoch
ckpt_callback = get_checkpoint_callback(
cpDir, 'val/acc', 'max',
filename='{epoch}_{val_acc:.2f}',
every_n_train_steps=every_n_train_steps)
# Logging callbacks
if args.train_scale >= 1:
version_str = f'{args.dataset}_ts={int(args.train_scale):d}'
else:
version_str = f'{args.dataset}_ts={args.train_scale:.2%}'
logger_tb = pl_log.TensorBoardLogger(args.log_dir, args.exp_name, version_str)
log_dir = logger_tb.log_dir
args.logger = [logger_tb]
if pl.utilities.distributed._get_rank() == 0:
os.makedirs(log_dir)
stream.all_to_file(log_dir+'/{}.log'.format(
args.exp_name), flush=True)
# logger_eren = MyLogger(log_dir, 'exp_log')
logger_eren = MyLogger(None)
args.progress_bar_refresh_rate = 0 # Use MyLogger() install of progress_bar
lr_monitor = pl.callbacks.LearningRateMonitor(logging_interval='epoch')
args.callbacks = [
ckpt_callback, logger_eren, lr_monitor
]
# * Accelerating *
if args.gpus > 1 and (args.accelerator is None and args.plugins is None):
args.accelerator = 'ddp'
if args.accelerator == 'ddp':
args.plugins = pl.plugins.DDPPlugin(find_unused_parameters=False)
if args.mode_name in ['train', 'finetune']:
args.benchmark = True
# * Begin training and testing *
trainer = Trainer.from_argparse_args(args)
# Begin training
trainer.fit(model, datamodule=dm)
# Final test
model.final_val = True
trainer.validate(model, ckpt_path='best', datamodule=dm)
# Other operations
print('Best ckpt: {}'.format(trainer.checkpoint_callback.best_model_path))
if args.ckpt != 'debug' and args.save_ckpt:
checkpoint_standardize(cpDir)
def test_init_with_array():
array = np.empty((50, 50))
with BasicModel(array) as dm:
assert dm.data.shape == (50, 50)
import shelve
import time
import matplotlib.pyplot as plt
# ========================================================================
# RUN SCRIPT
# ========================================================================
modelUsed = 'basic'
Data = scipy.io.loadmat('Data.mat') # it's a dictionary
Returns = Data['Returns']
S = Data['S']
J_jump = 0.07
r = 0.055
dt = 1 / 252
if modelUsed is 'basic':
model = BasicModel(J_jump, r, dt, S)
elif modelUsed is 'ext1':
LogReturns = np.log(1 + Returns)
model = GBMmodel(J_jump, r, dt, LogReturns)
N = 10
theta = 0.07
LB = 0.5
UB = 1.9
eta = 1e-3 / 5
X = [0] * (N + 1)
for k in range(1, N):
X[k] = np.arange(LB, UB + eta, eta)
X[0] = np.array([1])
X[-1] = np.arange((1 + theta)**2, UB + eta, eta)
# run ODAA algorithm
def test_dtype_match():
with BasicModel() as dm:
dm.data = np.array([[1, 2, 3]], np.float32)
