def test_skip_serializing_null(tmp_path, filename): """Make sure that None is not written out to the ASDF tree""" file_path = tmp_path / filename with BasicModel() as model: model.meta.telescope = None model.save(file_path) with BasicModel(file_path) as model: # Make sure that 'telescope' is not in the tree with pytest.raises(KeyError): assert model.meta["telescope"] is None
def test_stringify(tmp_path): dm = DataModel() assert str(dm) == '<DataModel>' dm = BasicModel((10, 100)) assert str(dm) == '<BasicModel(10, 100)>' file_path = tmp_path / "test.asdf" dm.save(file_path) dm.close() with BasicModel(file_path) as dm: assert str(dm) == '<BasicModel(10, 100) from test.asdf>'
def test_secondary_shapes(): """ Confirm that a non-primary array takes on the shape specified in the initializer. """ with BasicModel((256, 256)) as dm: assert dm.area.shape == (256, 256)
def test_broadcast2(): with BasicModel() as dm: data = np.empty((52, 50)) dm.data = data dq = np.empty((50, )) dm.dq = dq
def test_object_node_iterator(): m = BasicModel({"meta": {"foo": "bar"}}) items = [] for i in m.meta.items(): items.append(i[0]) assert 'foo' in items
def test_getarray_noinit_noinit(): """Test that calling on a non-existant array does not initialize that array""" arr = np.ones((5, 5)) model = BasicModel(data=arr) try: model.getarray_noinit('area') except AttributeError: pass assert 'area' not in model.instance
def test_update_from_dict(tmp_path): """Test update method from a dictionary""" file_path = tmp_path / "update.asdf" with BasicModel((5, 5)) as m: m.update({"foo": "bar", "baz": 42}) m.save(file_path) with asdf.open(file_path) as af: assert af["foo"] == "bar" assert af["baz"] == 42
def test_init_from_pathlib(tmp_path): """Test initializing model from a PurePath object""" file_path = tmp_path / "test.asdf" with asdf.AsdfFile() as af: af["meta"] = {"telescope": "crystal ball"} af.write_to(file_path) model = BasicModel(file_path) # Test is basically, did we open the model? assert model.meta.telescope == "crystal ball"
def test_copy(): with BasicModel((50, 50)) as dm: dm.meta.telescope = "MEADE" dm.meta.foo = "BAR" with dm.copy() as dm2: dm2.data[0, 0] = 42 assert np.sum(dm.data.flatten()) == 0 assert dm2.meta.telescope == "MEADE" assert dm2.meta.foo == "BAR" dm2.meta.foo = "BAZ" assert dm.meta.foo == "BAR" dm2.meta.origin = "STScI" assert dm.meta.origin is None
def evaluate_line(): with open(FLAGS.map_file, 'rb') as f: char_to_id, id_to_char, tag_to_id, id_to_tag = cPickle.load(f) config=BasicModelConfig(FLAGS,len(char_to_id),len(tag_to_id),4) tfConfig = tf.ConfigProto() tfConfig.gpu_options.per_process_gpu_memory_fraction = FLAGS.memory_usage with tf.Session(config=tfConfig) as sess: model = BasicModel(config, None) saver = tf.train.Saver() model_name = os.path.join(FLAGS.checkpoints, FLAGS.model_name) saver.restore(sess, model_name) while True: line = raw_input('\n请输入测试句子(0:Exit):\n'.encode('utf-8')).decode('utf-8') if line=='0': exit() strings, predicts, _ = model.evaluate_step(sess, input_from_line(line, char_to_id)) string=strings[0] tags=[id_to_tag[t] for t in predicts[0]] assert len(string)==len(tags) result_json=result_to_json(string,tags) print_result_json(result_json)
def test_init_with_array(): array = np.empty((50, 50)) with BasicModel(array) as dm: assert dm.data.shape == (50, 50)
def test_dtype_match(): with BasicModel() as dm: dm.data = np.array([[1, 2, 3]], np.float32)
def test_array_type(): with BasicModel() as dm: assert dm.dq.dtype == np.uint32
e2_classes=E2_CLASSES, model_id=model_id, input_dim=n_samples, out_dim=model_out_dim, ) elif isMode(mode, 'e1'): model = E1Model( classes=E1_CLASSES, model_id=model_id, input_dim=n_samples, out_dim=model_out_dim, ) elif isMode(mode, 'r'): model = BasicModel( input_dim=n_samples, out_dim=model_out_dim, model_id=model_id, ) print("Model {}".format(model_id)) # Prepare Data for prediction if isMode(mode, 'e1_e2'): pc = PermittivityCalculator() e1_classes, e2_classes = list(set(r_e1_e2_data['e1_cls'])), list( set(r_e1_e2_data['e2_cls'])) e1e2_prediction_classes = [ f'{e1_cls},{e2_cls}' for e1_cls in e1_classes for e2_cls in e2_classes ]
def test_set_shape(): with BasicModel((50, 50)) as dm: assert dm.shape == (50, 50) with pytest.raises(AttributeError): dm.shape = (42, 23)
def test_getarray_noinit_raises(): """Test for error when accessing non-existent array""" arr = np.ones((5, 5)) model = BasicModel(data=arr) with pytest.raises(AttributeError): model.getarray_noinit('area')
def test_getarray_noinit_valid(): """Test for valid value return""" arr = np.ones((5, 5)) model = BasicModel(data=arr) fetched = model.getarray_noinit('data') assert (fetched == arr).all()
def test_init_with_array2(): with pytest.raises(ValueError): array = np.empty((50, )) with BasicModel(array) as dm: dm.data
def test_search_schema(): with BasicModel() as x: results = x.search_schema('origin') assert [x[0] for x in results] == ['meta.origin']
def test_set_array(): with pytest.raises(ValueError): with BasicModel() as dm: data = np.empty((50, )) dm.data = data
dynamic_utility_parameter_distribution = True else: dynamic_utility_parameter_distribution = False sampling_policy_name = 'uEI_prior' elif sampling_policy_name is 'uTS': model = MultiOutputGP(output_dim=m, exact_feval=[True] * m, fixed_hyps=False) # Model (Multi-output GP) sampling_policy = uTS(model, space, optimizer='CMA', utility=utility) if learn_preferences: dynamic_utility_parameter_distribution = True else: dynamic_utility_parameter_distribution = False sampling_policy_name = 'uTS_prior' elif sampling_policy_name is 'Random': model = BasicModel(output_dim=m) sampling_policy = Random(model=None, space=space) dynamic_utility_parameter_distribution = False elif sampling_policy_name is 'ParEGO': model = BasicModel(output_dim=m) sampling_policy = ParEGO(model, space, utility) dynamic_utility_parameter_distribution = False # BO model max_iter = 150 experiment_name = 'test_dtlz1a' if len(sys.argv) > 1: experiment_number = int(sys.argv[1]) filename = [ experiment_name, sampling_policy_name, str(experiment_number)
def train(): # load data sets train_sentences=load_sentences(FLAGS.train_file,FLAGS.zeros) dev_sentences=load_sentences(FLAGS.dev_file,FLAGS.zeros) test_sentences=load_sentences(FLAGS.test_file,FLAGS.zeros) # appoint tagging scheme (IOB/IOBES) train_sentences=update_tag_scheme(train_sentences,FLAGS.tag_schema) dev_sentences=update_tag_scheme(dev_sentences,FLAGS.tag_schema) test_sentences=update_tag_scheme(test_sentences,FLAGS.tag_schema) #create maps if not exist if not os.path.exists(FLAGS.map_file): if FLAGS.pre_emb: char_to_id,_=char_mapping(train_sentences) char_to_id,id_to_char=augment_with_pretrained(char_to_id,'wiki_100.utf8') else: char_to_id, id_to_char=char_mapping(train_sentences) tag_to_id, id_to_tag=tag_mapping(train_sentences) with open(FLAGS.map_file,'wb') as f: cPickle.dump([char_to_id,id_to_char,tag_to_id,id_to_tag],f,cPickle.HIGHEST_PROTOCOL) else: with open(FLAGS.map_file,'rb') as f: char_to_id, id_to_char, tag_to_id, id_to_tag=cPickle.load(f) # prepare data, get a collection of list containing index train_data=prepare_dataset(train_sentences,char_to_id,tag_to_id,True) dev_data=prepare_dataset(dev_sentences,char_to_id,tag_to_id,True) test_data=prepare_dataset(test_sentences,char_to_id,tag_to_id,True) print "%i %i %i sentences in train / dev / test." % (len(train_data),len(dev_data),len(test_data)) if not FLAGS.pre_emb: pre_emb=None else: pre_emb=load_word2vec(FLAGS.pre_emb_file,char_to_id,FLAGS.char_dim) print "init embedding shape: (%d,%d)" %(pre_emb.shape[0],pre_emb.shape[1]) train_manager=BatchManager(train_data,FLAGS.batch_size,True) dev_manager=BatchManager(dev_data,FLAGS.batch_size,False) test_manager=BatchManager(test_data,FLAGS.batch_size,False) config=BasicModelConfig(FLAGS,len(char_to_id),len(tag_to_id),4) tfConfig = tf.ConfigProto() tfConfig.gpu_options.per_process_gpu_memory_fraction = FLAGS.memory_usage with tf.Session(config=tfConfig) as sess: print "Train started!" model=BasicModel(config,pre_emb) saver=tf.train.Saver() # tensorboard if not os.path.exists(FLAGS.summaries_dir): os.mkdir(FLAGS.summaries_dir) merged=tf.summary.merge_all() train_writer=tf.summary.FileWriter(os.path.join(FLAGS.summaries_dir,FLAGS.model_name,"train"),sess.graph) test_writer=tf.summary.FileWriter(os.path.join(FLAGS.summaries_dir,FLAGS.model_name,"test"),sess.graph) # load previous trained model or create a new model if not os.path.exists(FLAGS.checkpoints): os.mkdir(FLAGS.checkpoints) model_name=os.path.join(FLAGS.checkpoints,FLAGS.model_name) ckpt=tf.train.get_checkpoint_state(FLAGS.checkpoints) if ckpt and ckpt.model_checkpoint_path: print "restore from previous traied model: %s" % FLAGS.model_name saver.restore(sess,ckpt.model_checkpoint_path) else: sess.run(tf.global_variables_initializer()) def evaluate(sess,model,manager): strings=[] predicts=[] goldens=[] bar = ProgressBar(max_value=manager.num_batch) for batch in bar(manager.iter_batch()): batch_string,batch_predict,batch_golden=model.evaluate_step(sess,batch) strings.extend(batch_string) predicts.extend(batch_predict) goldens.extend(batch_golden) return strings,predicts,goldens best_eval_f1=0 noimpro_num=0 for i in range(FLAGS.max_epoch): #train train_loss=[] bar = ProgressBar(max_value=train_manager.num_batch) for step,batch in bar(enumerate(train_manager.iter_batch())): batch.append(merged) summary,global_step,batch_loss=model.train_step(sess,batch,FLAGS.dropout_keep) #add summary to tensorboard train_writer.add_summary(summary,global_step) train_loss.append(batch_loss) print "Epoch %d Train loss is %.4f" % (i+1,np.mean(train_loss)) #dev strings,predicts,goldens=evaluate(sess,model,dev_manager) eval_f1=report_results(strings,predicts,goldens,id_to_char,id_to_tag,'outputs/dev') if eval_f1>best_eval_f1: best_eval_f1=eval_f1 noimpro_num=0 saver.save(sess,model_name) else: noimpro_num+=1 print "Epoch %d Best eval f1:%.6f" % (i+1,best_eval_f1) #test strings,predicts,goldens=evaluate(sess,model,test_manager) test_f1=report_results(strings,predicts,goldens,id_to_char,id_to_tag,'outputs/test',True) #early_stop if noimpro_num>=3: print "Early stop! Final F1 scores on test data is :%.6f" % test_f1 break print
def test_initialize_arrays_with_arglist(): shape = (10, 10) area = np.full((2, 2), 13.0) m = BasicModel(shape, area=area) assert np.array_equal(m.area, area)
def test_implicit_creation_lower_dimensionality(): with BasicModel(np.zeros((10, 20))) as m: assert m.dq.shape == (20, )
def test_broadcast(): with BasicModel((50, 50)) as dm: data = np.empty((50, )) dm.dq = data
def main(args, stream): if not args.class_wise_sampling and args.data_seed is None: args.data_seed = random.randint(1, 1e8) # * Prepare data_module * dm = DInterface(**vars(args)) args.class_dict = dm.init_data['class_dict'] args.classes = list(args.class_dict.keys()) args.num_classes = len(args.class_dict) global_bs = args.gpus * args.batch_size if args.gpus > 1 else args.batch_size # * Build model * net = build_model(**vars(args)) if args.load_pretrained: pretrained_path = load_pretrain_path_by_args(args, '.pth.tar') bl_layers = None if args.mode_name in ['train', 'finetune']: bl_layers = ['classifier', 'fc'] net = load_ckpt(net, pretrained_path, train=(args.mode_name == 'train'), block_layers=bl_layers, map_keys=args.map_keys, verbose=True) model = BasicModel(net, **vars(args)) # Resume load_path = load_model_path_by_args(args) if load_path is not None: model.load_from_checkpoint(checkpoint_path=load_path, strict=False) # * validate mode * if args.mode_name in ['val', 'test']: model.final_val = True trainer = Trainer.from_argparse_args(args) trainer.validate(model, datamodule=dm) return # * Callbacks * # Checkpoint callbacks if args.ckpt == 'debug' or not args.save_ckpt: # ckpt_callback = get_checkpoint_callback(args.ckpt, save_last=False, save_top_k=0) ckpt_callback = get_checkpoint_callback(f'Task_models/{args.net_suffix}', save_last=False, save_top_k=1) else: cpDir = '{}/{}_{}'.format(args.ckpt, args.model_name, args.net_suffix) every_n_train_steps = dm.num_samples//global_bs if args.ckpt_ever_n_epoch: every_n_train_steps *= args.ckpt_ever_n_epoch ckpt_callback = get_checkpoint_callback( cpDir, 'val/acc', 'max', filename='{epoch}_{val_acc:.2f}', every_n_train_steps=every_n_train_steps) # Logging callbacks if args.train_scale >= 1: version_str = f'{args.dataset}_ts={int(args.train_scale):d}' else: version_str = f'{args.dataset}_ts={args.train_scale:.2%}' logger_tb = pl_log.TensorBoardLogger(args.log_dir, args.exp_name, version_str) log_dir = logger_tb.log_dir args.logger = [logger_tb] if pl.utilities.distributed._get_rank() == 0: os.makedirs(log_dir) stream.all_to_file(log_dir+'/{}.log'.format( args.exp_name), flush=True) # logger_eren = MyLogger(log_dir, 'exp_log') logger_eren = MyLogger(None) args.progress_bar_refresh_rate = 0 # Use MyLogger() install of progress_bar lr_monitor = pl.callbacks.LearningRateMonitor(logging_interval='epoch') args.callbacks = [ ckpt_callback, logger_eren, lr_monitor ] # * Accelerating * if args.gpus > 1 and (args.accelerator is None and args.plugins is None): args.accelerator = 'ddp' if args.accelerator == 'ddp': args.plugins = pl.plugins.DDPPlugin(find_unused_parameters=False) if args.mode_name in ['train', 'finetune']: args.benchmark = True # * Begin training and testing * trainer = Trainer.from_argparse_args(args) # Begin training trainer.fit(model, datamodule=dm) # Final test model.final_val = True trainer.validate(model, ckpt_path='best', datamodule=dm) # Other operations print('Best ckpt: {}'.format(trainer.checkpoint_callback.best_model_path)) if args.ckpt != 'debug' and args.save_ckpt: checkpoint_standardize(cpDir)
def test_delete(): with BasicModel() as dm: dm.meta.telescope = 'JWST' assert dm.meta.telescope == 'JWST' del dm.meta.telescope assert dm.meta.telescope is None
import shelve import time import matplotlib.pyplot as plt # ======================================================================== # RUN SCRIPT # ======================================================================== modelUsed = 'basic' Data = scipy.io.loadmat('Data.mat') # it's a dictionary Returns = Data['Returns'] S = Data['S'] J_jump = 0.07 r = 0.055 dt = 1 / 252 if modelUsed is 'basic': model = BasicModel(J_jump, r, dt, S) elif modelUsed is 'ext1': LogReturns = np.log(1 + Returns) model = GBMmodel(J_jump, r, dt, LogReturns) N = 10 theta = 0.07 LB = 0.5 UB = 1.9 eta = 1e-3 / 5 X = [0] * (N + 1) for k in range(1, N): X[k] = np.arange(LB, UB + eta, eta) X[0] = np.array([1]) X[-1] = np.arange((1 + theta)**2, UB + eta, eta) # run ODAA algorithm
def test_set_array2(): with BasicModel() as dm: data = np.empty((50, 50)) dm.data = data