예제 #1
0
    def __init__(self,
                 net,
                 snapshot_prefix,
                 dp_params,
                 preproc,
                 iter=None,
                 test=False):
        if iter is not None:
            self.N = caffe.Net(net,
                               snapshot_prefix + str(iter) + '.caffemodel',
                               caffe.TRAIN)
            self.iter = iter
        else:
            if test:
                self.N = caffe.Net(net, snapshot_prefix, caffe.TEST)
#                self.N = caffe.Net(net, caffe.TRAIN)
            else:
                self.N = caffe.Net(net, snapshot_prefix, caffe.TRAIN)
            self.iter = 0

        # Data provider
        self.dp = DataProvider(dp_params, preproc)
        self.bsize = self.dp.batch_size

        self.prevs = {}
        self.test = test
예제 #2
0
 def init_data_providers(self, dp_type=None):
     if dp_type is not None:
         self.dp_type = dp_type
         self.train_batch_range = self.test_batch_range
     self.dp_params['convnet'] = self
     self.dp_params['img_size'] = self.img_size
     try:
         self.test_data_provider = DataProvider.get_instance(
             self.data_path,
             self.test_batch_range,
             type=self.dp_type,
             dp_params=self.dp_params,
             test=True)
         self.train_data_provider = DataProvider.get_instance(
             self.data_path,
             self.train_batch_range,
             self.model_state["epoch"],
             self.model_state["batchnum"],
             type=self.dp_type,
             dp_params=self.dp_params,
             test=False)
     except DataProviderException, e:
         print "Unable to create data provider: %s" % e
         self.print_data_providers()
         sys.exit()
예제 #3
0
파일: da_main.py 프로젝트: he-org/CLEIT
def main(args):
    if args.method == 'dann':
        train_fn = train_dann.train_dann
    elif args.method == 'adda':
        train_fn = train_adda.train_adda
    elif args.method == 'dcc':
        train_fn = train_dcc.train_dcc
    elif args.method == 'cleitcs':
        train_fn = train_cleitcs.train_cleitcs
    else:
        train_fn = train_coral.train_coral

    device = 'cuda' if torch.cuda.is_available() else 'cpu'

    with open(os.path.join('model_save', 'train_params.json'), 'r') as f:
        training_params = json.load(f)

    training_params.update(
        {
            'device': device,
            'model_save_folder': os.path.join('model_save', args.method, 'labeled'),
        })

    safe_make_dir(training_params['model_save_folder'])
    data_provider = DataProvider(batch_size=training_params['labeled']['batch_size'],
                                 target=args.measurement)
    training_params.update(
        {
            'input_dim': data_provider.shape_dict['gex'],
            'output_dim': data_provider.shape_dict['target']
        }
    )

    s_labeled_dataloader = data_provider.get_labeled_gex_dataloader()
    labeled_dataloader_generator = data_provider.get_drug_labeled_mut_dataloader()

    s_ft_evaluation_metrics = defaultdict(list)
    t_ft_evaluation_metrics = defaultdict(list)
    val_ft_evaluation_metrics = defaultdict(list)
    test_ft_evaluation_metrics = defaultdict(list)

    fold_count = 0
    for train_labeled_dataloader, val_labeled_dataloader, test_labeled_dataloader in labeled_dataloader_generator:
        target_regressor, train_historys = train_fn(s_dataloaders=s_labeled_dataloader,
                                                    t_dataloaders=train_labeled_dataloader,
                                                    val_dataloader=val_labeled_dataloader,
                                                    test_dataloader=test_labeled_dataloader,
                                                    metric_name=args.metric,
                                                    seed = fold_count,
                                                    **wrap_training_params(training_params, type='labeled'))

        for metric in ['dpearsonr', 'dspearmanr', 'drmse', 'cpearsonr', 'cspearmanr', 'crmse']:
            val_ft_evaluation_metrics[metric].append(train_historys[-2][metric][train_historys[-2]['best_index']])
            test_ft_evaluation_metrics[metric].append(train_historys[-1][metric][train_historys[-2]['best_index']])
        fold_count += 1

    with open(os.path.join(training_params['model_save_folder'], f'test_ft_evaluation_results.json'), 'w') as f:
        json.dump(test_ft_evaluation_metrics, f)
    with open(os.path.join(training_params['model_save_folder'], f'ft_evaluation_results.json'), 'w') as f:
        json.dump(val_ft_evaluation_metrics, f)
 def init_data_providers(self):
     self.dp_params['convnet'] = self
     try:
         self.test_data_provider = DataProvider.get_instance(
             self.data_path,
             self.img_size,
             self.img_channels,  # options i've add to cifar data provider
             self.test_batch_range,
             type=self.dp_type,
             dp_params=self.dp_params,
             test=True)
         self.train_data_provider = DataProvider.get_instance(
             self.data_path,
             self.img_size,
             self.img_channels,  # options i've add to cifar data provider
             self.train_batch_range,
             self.model_state["epoch"],
             self.model_state["batchnum"],
             type=self.dp_type,
             dp_params=self.dp_params,
             test=False)
     except DataProviderException, e:
         print "Unable to create data provider: %s" % e
         self.print_data_providers()
         sys.exit()
예제 #5
0
파일: mlp_main.py 프로젝트: he-org/CLEIT
def main(args):
    device = 'cuda' if torch.cuda.is_available() else 'cpu'
    with open(os.path.join('model_save', 'train_params.json'), 'r') as f:
        training_params = json.load(f)

    training_params.update(
        {
            'device': device,
            'model_save_folder': os.path.join('model_save', 'mlp', args.omics),
        })

    data_provider = DataProvider(batch_size=training_params['unlabeled']['batch_size'],
                                 target=args.measurement)

    training_params.update(
        {
            'input_dim': data_provider.shape_dict[args.omics],
            'output_dim': data_provider.shape_dict['target']
        }
    )

    ft_evaluation_metrics = defaultdict(list)
    if args.omics == 'gex':
        labeled_dataloader_generator = data_provider.get_drug_labeled_gex_dataloader()
        fold_count = 0
        for train_labeled_dataloader, val_labeled_dataloader in labeled_dataloader_generator:
            target_regressor, ft_historys = fine_tune_encoder(
                train_dataloader=train_labeled_dataloader,
                val_dataloader=val_labeled_dataloader,
                test_dataloader=val_labeled_dataloader,
                seed=fold_count,
                metric_name=args.metric,
                **wrap_training_params(training_params, type='labeled')
            )
            for metric in ['dpearsonr', 'dspearmanr','drmse', 'cpearsonr', 'cspearmanr','crmse']:
                ft_evaluation_metrics[metric].append(ft_historys[-2][metric][ft_historys[-2]['best_index']])
            fold_count += 1
    else:
        labeled_dataloader_generator = data_provider.get_drug_labeled_mut_dataloader()
        fold_count = 0
        test_ft_evaluation_metrics = defaultdict(list)

        for train_labeled_dataloader, val_labeled_dataloader, test_labeled_dataloader in labeled_dataloader_generator:
            target_regressor, ft_historys = fine_tune_encoder(
                train_dataloader=train_labeled_dataloader,
                val_dataloader=val_labeled_dataloader,
                test_dataloader=test_labeled_dataloader,
                seed=fold_count,
                metric_name=args.metric,
                **wrap_training_params(training_params, type='labeled')
            )
            for metric in ['dpearsonr', 'dspearmanr','drmse', 'cpearsonr', 'cspearmanr','crmse']:
                ft_evaluation_metrics[metric].append(ft_historys[-2][metric][ft_historys[-2]['best_index']])
                test_ft_evaluation_metrics[metric].append(ft_historys[-1][metric][ft_historys[-2]['best_index']])
            fold_count += 1
        with open(os.path.join(training_params['model_save_folder'], f'test_ft_evaluation_results.json'), 'w') as f:
            json.dump(test_ft_evaluation_metrics, f)
    with open(os.path.join(training_params['model_save_folder'], f'ft_evaluation_results.json'), 'w') as f:
        json.dump(ft_evaluation_metrics, f)
예제 #6
0
파일: trainer.py 프로젝트: iskandr/striate
 def init_subnet_data_provider(self):
   if self.output_method == 'disk':
     dp = DataProvider.get_by_name('intermediate')
     count = self.train_dumper.get_count()
     self.train_dp = dp(self.train_output_filename,  range(0, count), 'fc')
     count = self.test_dumper.get_count()
     self.test_dp = dp(self.test_output_filename, range(0, count), 'fc')
   elif self.output_method == 'memory':
     dp = DataProvider.get_by_name('memory')
     self.train_dp = dp(self.train_dumper)
     self.test_dp = dp(self.test_dumper)
예제 #7
0
 def init_subnet_data_provider(self):
     if self.output_method == 'disk':
         dp = DataProvider.get_by_name('intermediate')
         count = self.train_dumper.get_count()
         self.train_dp = dp(self.train_output_filename, range(0, count),
                            'fc')
         count = self.test_dumper.get_count()
         self.test_dp = dp(self.test_output_filename, range(0, count), 'fc')
     elif self.output_method == 'memory':
         dp = DataProvider.get_by_name('memory')
         self.train_dp = dp(self.train_dumper)
         self.test_dp = dp(self.test_dumper)
 def init_data_providers(self):
     self.dp_params['convnet'] = self # dp aka dataprovider
     try:
         self.test_data_provider = DataProvider.get_instance(self.data_path, self.test_batch_range,
                                                             type=self.dp_type, dp_params=self.dp_params, test=True)
         self.train_data_provider = DataProvider.get_instance(self.data_path, self.train_batch_range,
                                                                  self.model_state["epoch"], self.model_state["batchnum"],
                                                                  type=self.dp_type, dp_params=self.dp_params, test=False)
     except DataProviderException, e:
         print "Unable to create data provider: %s" % e
         self.print_data_providers()
         sys.exit()
예제 #9
0
파일: trainer.py 프로젝트: wqren/striate
 def set_category_range(self, r):
     dp = DataProvider.get_by_name(self.data_provider)
     self.train_dp = dp(self.data_dir,
                        self.train_range,
                        category_range=range(r))
     self.test_dp = dp(self.data_dir,
                       self.test_range,
                       category_range=range(r))
예제 #10
0
 def init_data_providers(self):
     self.dp_params['convnet'] = self
     self.dp_params['PCA_pixel_alter'] = self.PCA_pixel_alter        
     self.dp_params['regress_L_channel_only'] = self.regress_L_channel_only
     self.dp_params['use_local_context_ftr'] = self.use_local_context_ftr
     self.dp_params['use_local_context_color_ftr'] = self.use_local_context_color_ftr
     if hasattr(self,'use_position_ftr'):
         self.dp_params['use_position_ftr'] = self.use_position_ftr
     try:
         self.test_data_provider = DataProvider.get_instance(self.libmodel, self.data_path, self.test_batch_range,
                                                             type=self.dp_type, dp_params=self.dp_params, test=DataProvider.DP_TEST)
         self.train_data_provider = DataProvider.get_instance(self.libmodel, self.data_path, self.train_batch_range,
                                                                  self.model_state["epoch"], self.model_state["batch_idx"],
                                                                  self.model_state["epochBatchPerm"],
                                                                  type=self.dp_type, dp_params=self.dp_params, test=DataProvider.DP_TRAIN)
     except DataProviderException, e:
         print "Unable to create data provider: %s" % e
         self.print_data_providers()
         sys.exit()
예제 #11
0
 def init_data_providers(self):
     class Dummy:
         def advance_batch(self):
             pass
     if self.need_gpu:
         ConvNet.init_data_providers(self)
         if self.op.get_value("write_features_pred") or self.op.get_value("show_preds") == 2:
             self.pred_data_provider = DataProvider.get_instance(self.libmodel, self.data_path, self.pred_batch_range,
                                                                 type=self.dp_type, dp_params=self.dp_params, test=DataProvider.DP_PREDICT)
         
     else:
         self.train_data_provider = self.test_data_provider = Dummy()
예제 #12
0
파일: gpumodel.py 프로젝트: lelegan/DLSR
    def init_data_providers(self):
        self.dp_params['convnet'] = self
        self.dp_params['imgprovider'] = self.img_provider_file
        try:
            self.test_data_provider = DataProvider.get_instance(self.data_path_test, self.test_batch_range,
                                                                type=self.dp_type_test, dp_params=self.dp_params, test=True)
            if not self.test_only:
                self.train_data_provider = DataProvider.get_instance(self.data_path_train, self.train_batch_range,
                                                                     self.model_state["epoch"], self.model_state["batchnum"],
                                                                     type=self.dp_type_train, dp_params=self.dp_params, test=False)

            self.test_batch_range = self.test_data_provider.batch_range
            print "Test data provider: ", len(self.test_batch_range), " batches "
            if not self.test_only:
                self.train_batch_range = self.train_data_provider.batch_range
                print "Training data provider: ", len(self.train_batch_range), " batches "

        except DataProviderException, e:
            print "Unable to create data provider: %s" % e
            self.print_data_providers()
            sys.exit()
예제 #13
0
파일: rnn_fwd.py 프로젝트: qrepa/PSIML
def main():
    hidden_units = 15
    outputs = 1
    inputs = 5
    examples = 100
    sequence_length = 10
    learning_rate = 0.001

    # Shapes of weights:
    # Input -> hidden connections
    # W_ih.shape = (inputs, hidden_units)
    # b_ih.shape = (hidden_units,)
    # Hidden -> hidden connections
    # W_hh.shape = (hidden_units, hidden_units)
    # Hidden -> output connections
    # W_hk.shape = (hidden_units, outputs)
    # b_hk.shape = (outputs,)

    # Load trained network.
    filename = r"model/trained_net.wts.npz"
    W_ih, b_ih, W_hh, W_hk, b_hk = load_params(filename)

    # Get training set.
    d = DataProvider(examples, sequence_length, inputs, outputs)
    # x.shape = (sequence_length, inputs)
    # z.shape = (sequence_length, 1)
    x, z = d.get_example(0)

    # dictionary where key is the timestamp.
    a_h = {}
    b_h = dict()
    b_h[-1] = np.zeros_like(b_ih)
    a_k = {}
    y = {}
    for t in range(sequence_length):
        a_h[t] = None  # TODO: (3.30), don't forget bias parameter
        b_h[t] = None  # TODO: (3.31), hint: theta_h = tanh
        a_k[t] = None  # TODO: (3.32), don't forget bias parameter
        y[t] = None  # TODO: Binary classification
    plot_predictions(y)
예제 #14
0
 def init_data_providers(self):
     self.dp_params['convnet'] = self
     self.dp_params['imgprovider'] = self.img_provider_file
     try:
         if self.need_gpu:
             self.test_data_provider = DataProvider.get_instance(self.data_path_test, self.test_batch_range,
                                                                 type=self.dp_type_test, dp_params=self.dp_params, test=True)
             
             self.test_batch_range = self.test_data_provider.batch_range
     except Exception, e:
         print "Unable to create data provider: %s" % e
         self.print_data_providers()
         sys.exit()
예제 #15
0
    def init_data_providers(self):
        class Dummy:
            def advance_batch(self):
                pass

        if self.need_gpu:
            ConvNet.init_data_providers(self)
            if self.op.get_value("write_features_pred") or self.op.get_value(
                    "show_preds") == 2:
                self.pred_data_provider = DataProvider.get_instance(
                    self.libmodel,
                    self.data_path,
                    self.pred_batch_range,
                    type=self.dp_type,
                    dp_params=self.dp_params,
                    test=DataProvider.DP_PREDICT)

        else:
            self.train_data_provider = self.test_data_provider = Dummy()
예제 #16
0
def test_gan(gan_type,
             g_input,
             g_output,
             d_input,
             d_output,
             epochs=3,
             verbose=False):
    # we have to reinstantiate the generator and discriminator or the weights won't be reset
    generator = Model(g_input, g_output, name='generator')
    discriminator = Model(d_input, d_output, name='discriminator')
    gan_model = gan_type(generator, discriminator)
    gan_model.num_data = len(metadata)
    gan_model.compile()
    data_train = DataProvider(metadata_train)
    # build a dictionary mapping between name strings and ids
    data_train.class_to_id = dict((n, i) for i, n in enumerate(classes))
    data_train.id_to_class = dict((i, n) for i, n in enumerate(classes))
    data_test = DataProvider(metadata_test)
    data_test.class_to_id = dict((n, i) for i, n in enumerate(classes))
    data_test.id_to_class = dict((i, n) for i, n in enumerate(classes))
    gan_model.train(epochs, data_train, data_test, verbose=verbose)
예제 #17
0
파일: trainer.py 프로젝트: wqren/striate

  #create a checkpoint dumper
  image_shape = (param_dict['image_color'], param_dict['image_size'], param_dict['image_size'], param_dict['batch_size'])
  param_dict['image_shape'] = image_shape
  cp_dumper = CheckpointDumper(param_dict['checkpoint_dir'], param_dict['test_id'])
  param_dict['checkpoint_dumper'] = cp_dumper

  #create the init_model
  init_model = cp_dumper.get_checkpoint()
  if init_model is None:
    init_model = parse_config_file(args.param_file)
  param_dict['init_model'] = init_model

  #create train dataprovider and test dataprovider
  dp_class = DataProvider.get_by_name(param_dict['data_provider'])
  train_dp = dp_class(param_dict['data_dir'], param_dict['train_range'])
  test_dp = dp_class(param_dict['data_dir'], param_dict['test_range'])
  param_dict['train_dp'] = train_dp
  param_dict['test_dp'] = test_dp


  #get all extra information
  param_dict['num_epoch'] = args.num_epoch
  num_batch = util.string_to_int_list(args.num_batch)
  if len(num_batch) == 1:
    param_dict['num_batch'] = num_batch[0]
  else:
    param_dict['num_batch'] = num_batch

  param_dict['num_group_list']  = util.string_to_int_list(args.num_group_list)
예제 #18
0
os.environ['THEANO_FLAGS'] = 'device=gpu'

import os
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"  # see issue #152
os.environ["CUDA_VISIBLE_DEVICES"] = "1"

config = tf.ConfigProto(log_device_placement=True, allow_soft_placement=True)
config.gpu_options.allow_growth = True
session = tf.Session(config=config)
K.set_session(session)

conf = Config(flag, args[2], int(args[3]))
print(flag)

# get data
dp = DataProvider(conf)
n_terms = len(dp.idx2word)
word_embed_data = np.array(dp.word_embed)

item_embed_data = np.random.rand(dp.get_item_size(), conf.dim_word)
print("finish data processing")

# define model
word_input = Input(shape=(1, ), dtype="int32", name="word_idx")
item_pos_input = Input(shape=(1, ), dtype="int32", name="item_pos_idx")
item_neg_input = Input(shape=(1, ), dtype="int32", name="item_neg_idx")

word_embed = Embedding(output_dim=conf.dim_word,
                       input_dim=n_terms,
                       input_length=1,
                       name="word_embed",
예제 #19
0
                for i in range(train_y_truths.shape[0])
            ]).item())
        train_history['drmse'].append(
            np.mean(
                np.nanmean(np.square((train_y_truths - train_y_preds)),
                           axis=0)).item())
        train_history['crmse'].append(
            np.mean(
                np.nanmean(np.square((train_y_truths - train_y_preds)),
                           axis=1)).item())

    return (train_history, test_history)


if __name__ == '__main__':
    data_provider = DataProvider()
    labeled_samples, mut_only_labeled_samples = data_provider.get_labeled_samples(
    )
    labeled_target_df = data_provider.target_df.loc[labeled_samples]
    labeled_mut_only_target_df = data_provider.target_df.loc[
        mut_only_labeled_samples]
    label_gex_df = data_provider.gex_dat.loc[labeled_samples]
    label_mut_df = data_provider.mut_dat.loc[labeled_samples]
    label_mut_only_df = data_provider.mut_dat.loc[mut_only_labeled_samples]

    train_gex_data = (label_gex_df, labeled_target_df)
    train_mut_data = (label_mut_df, labeled_target_df)
    test_data = (label_mut_only_df, labeled_mut_only_target_df)

    gex_train_history, _ = n_time_cv_regress(train_gex_data,
                                             'gex_pred',
예제 #20
0
        <span style="font-weight: bold;">Number_of_Casualties: </span>@Number_of_Casualties
    </div>
    <div>
        <span style="font-weight: bold;">Nearest Hospital: </span>@closest_hospital_name
    </div>
    <div>
        <span style="font-weight: bold;">Distance to @closest_hospital_name: </span>@closest_hospital_distance km
    </div>   
</div>
"""

COL_TPL = """
<%= get_icon(type.toLowerCase()) %> <%= type %>
"""

data_provider = DataProvider()

data_scr = data_provider.data_ds
[start_date_str, end_date_str] = data_provider.get_boundary_dates()

max_casualties = data_provider.get_max_casualties()

fa_formatter = HTMLTemplateFormatter(template=COL_TPL)
columns = [
    TableColumn(field="datetime",
                default_sort="descending",
                title="Time",
                formatter=DateFormatter(format="%Y-%m-%d %H:%M:%S")),
    TableColumn(field="Verbal_severity", title="Accident_Severity", width=150),
    TableColumn(field="Number_of_Casualties",
                title="Number_of_Casualties",
예제 #21
0
파일: trainer.py 프로젝트: wqren/striate
    #create a checkpoint dumper
    image_shape = (param_dict['image_color'], param_dict['image_size'],
                   param_dict['image_size'], param_dict['batch_size'])
    param_dict['image_shape'] = image_shape
    cp_dumper = CheckpointDumper(param_dict['checkpoint_dir'],
                                 param_dict['test_id'])
    param_dict['checkpoint_dumper'] = cp_dumper

    #create the init_model
    init_model = cp_dumper.get_checkpoint()
    if init_model is None:
        init_model = parse_config_file(args.param_file)
    param_dict['init_model'] = init_model

    #create train dataprovider and test dataprovider
    dp_class = DataProvider.get_by_name(param_dict['data_provider'])
    train_dp = dp_class(param_dict['data_dir'], param_dict['train_range'])
    test_dp = dp_class(param_dict['data_dir'], param_dict['test_range'])
    param_dict['train_dp'] = train_dp
    param_dict['test_dp'] = test_dp

    #get all extra information
    param_dict['num_epoch'] = args.num_epoch
    num_batch = util.string_to_int_list(args.num_batch)
    if len(num_batch) == 1:
        param_dict['num_batch'] = num_batch[0]
    else:
        param_dict['num_batch'] = num_batch

    param_dict['num_group_list'] = util.string_to_int_list(args.num_group_list)
    param_dict['num_caterange_list'] = util.string_to_int_list(
예제 #22
0
파일: trainer.py 프로젝트: wqren/striate
 def set_num_group(self, n):
   dp = DataProvider.get_by_name(self.data_provider)
   self.train_dp = dp(self.data_dir, self.train_range, n)
   self.test_dp = dp(self.data_dir, self.test_range, n)
예제 #23
0
def main(args, update_params_dict):
    train_fn = train_ae.train_ae
    device = 'cuda' if torch.cuda.is_available() else 'cpu'
    with open(os.path.join('model_save', 'train_params.json'), 'r') as f:
        training_params = json.load(f)

    training_params['unlabeled'].update(update_params_dict)
    param_str = dict_to_str(update_params_dict)
    training_params.update({
        'device': device,
        'es_flag': False,
        'retrain_flag': args.retrain_flag
    })

    if args.omics != 'both':
        training_params.update({
            'device':
            device,
            'model_save_folder':
            os.path.join('model_save', 'ae', args.omics, param_str),
            'es_flag':
            False,
            'retrain_flag':
            args.retrain_flag
        })
        safe_make_dir(training_params['model_save_folder'])

    task_save_folder = os.path.join('model_save', 'ae', args.omics, param_str)
    safe_make_dir(task_save_folder)

    random.seed(2020)

    data_provider = DataProvider(
        batch_size=training_params['unlabeled']['batch_size'],
        target=args.measurement)

    # if args.omics == 'both':
    #     training_params.update(
    #         {
    #             'input_dim': sum([data_provider.shape_dict[k] for k in data_provider.shape_dict if k != 'target']),
    #             'output_dim': data_provider.shape_dict['target']
    #         }
    #     )
    # else:
    training_params.update({'output_dim': data_provider.shape_dict['target']})
    if args.omics != 'both':
        training_params.update({
            'input_dim':
            data_provider.shape_dict[args.omics],
        })

    # start unlabeled training
    if args.omics == 'gex':
        encoder, historys = train_fn(
            dataloader=data_provider.get_unlabeled_gex_dataloader(),
            **wrap_training_params(training_params, type='unlabeled'))
        with open(
                os.path.join(training_params['model_save_folder'],
                             f'unlabel_train_history.pickle'), 'wb') as f:
            for history in historys:
                pickle.dump(dict(history), f)

    elif args.omics == 'mut':
        encoder, historys = train_fn(
            dataloader=data_provider.get_unlabeld_mut_dataloader(match=False),
            **wrap_training_params(training_params, type='unlabeled'))
        with open(
                os.path.join(training_params['model_save_folder'],
                             f'unlabel_train_history.pickle'), 'wb') as f:
            for history in historys:
                pickle.dump(dict(history), f)
    else:
        training_params.update({
            'model_save_folder':
            os.path.join('model_save', 'ae', 'gex', param_str),
            'input_dim':
            data_provider.shape_dict['gex'],
        })
        safe_make_dir(training_params['model_save_folder'])

        gex_encoder, gex_historys = train_fn(
            dataloader=data_provider.get_unlabeled_gex_dataloader(),
            **wrap_training_params(training_params, type='unlabeled'))

        training_params.update({
            'model_save_folder':
            os.path.join('model_save', 'ae', 'mut', param_str),
            'input_dim':
            data_provider.shape_dict['mut'],
        })
        safe_make_dir(training_params['model_save_folder'])

        mut_encoder, mut_historys = train_fn(
            dataloader=data_provider.get_unlabeld_mut_dataloader(match=False),
            **wrap_training_params(training_params, type='unlabeled'))

    ft_evaluation_metrics = defaultdict(list)
    fold_count = 0
    if args.omics == 'gex':
        labeled_dataloader_generator = data_provider.get_labeled_data_generator(
            omics='gex')
        for train_labeled_dataloader, val_labeled_dataloader in labeled_dataloader_generator:
            ft_encoder = deepcopy(encoder)
            target_regressor, ft_historys = fine_tuning.fine_tune_encoder(
                encoder=ft_encoder,
                train_dataloader=train_labeled_dataloader,
                val_dataloader=val_labeled_dataloader,
                test_dataloader=val_labeled_dataloader,
                seed=fold_count,
                metric_name=args.metric,
                task_save_folder=task_save_folder,
                **wrap_training_params(training_params, type='labeled'))
            for metric in [
                    'dpearsonr', 'dspearmanr', 'drmse', 'cpearsonr',
                    'cspearmanr', 'crmse'
            ]:
                ft_evaluation_metrics[metric].append(
                    ft_historys[-2][metric][ft_historys[-2]['best_index']])
            fold_count += 1
    elif args.omics == 'mut':
        labeled_dataloader_generator = data_provider.get_labeled_data_generator(
            omics='mut')
        test_ft_evaluation_metrics = defaultdict(list)
        for train_labeled_dataloader, val_labeled_dataloader, test_labeled_dataloader in labeled_dataloader_generator:
            ft_encoder = deepcopy(encoder)
            target_regressor, ft_historys = fine_tuning.fine_tune_encoder(
                encoder=ft_encoder,
                train_dataloader=train_labeled_dataloader,
                val_dataloader=val_labeled_dataloader,
                test_dataloader=test_labeled_dataloader,
                seed=fold_count,
                metric_name=args.metric,
                task_save_folder=task_save_folder,
                **wrap_training_params(training_params, type='labeled'))
            for metric in [
                    'dpearsonr', 'dspearmanr', 'drmse', 'cpearsonr',
                    'cspearmanr', 'crmse'
            ]:
                ft_evaluation_metrics[metric].append(
                    ft_historys[-2][metric][ft_historys[-2]['best_index']])
                test_ft_evaluation_metrics[metric].append(
                    ft_historys[-1][metric][ft_historys[-2]['best_index']])
            fold_count += 1
        with open(
                os.path.join(task_save_folder,
                             f'{param_str}_test_ft_evaluation_results.json'),
                'w') as f:
            json.dump(test_ft_evaluation_metrics, f)

    else:
        labeled_dataloader_generator = data_provider.get_labeled_data_generator(
            omics='both')
        for train_labeled_dataloader, val_labeled_dataloader in labeled_dataloader_generator:
            ft_gex_encoder = deepcopy(gex_encoder)
            ft_mut_encoder = deepcopy(mut_encoder)
            target_regressor, ft_historys = fine_tuning_mul.fine_tune_encoder(
                encoders=[ft_gex_encoder, ft_mut_encoder],
                train_dataloader=train_labeled_dataloader,
                val_dataloader=val_labeled_dataloader,
                test_dataloader=val_labeled_dataloader,
                seed=fold_count,
                metric_name=args.metric,
                task_save_folder=task_save_folder,
                **wrap_training_params(training_params, type='labeled'))
            for metric in [
                    'dpearsonr', 'dspearmanr', 'drmse', 'cpearsonr',
                    'cspearmanr', 'crmse'
            ]:
                ft_evaluation_metrics[metric].append(
                    ft_historys[-2][metric][ft_historys[-2]['best_index']])
            fold_count += 1

    with open(
            os.path.join(task_save_folder,
                         f'{param_str}_ft_evaluation_results.json'), 'w') as f:
        json.dump(ft_evaluation_metrics, f)
예제 #24
0
def main():
    global cfg, global_all_step
    global_all_step = 0
    cfg = DefaultConfig()
    args = {
        'resnet18': RESNET18_SUNRGBD_CONFIG().args(),
    }

    # Setting random seed
    # if cfg.MANUAL_SEED is None:
    #     cfg.MANUAL_SEED = random.randint(1, 10000)
    # random.seed(cfg.MANUAL_SEED)
    # torch.manual_seed(cfg.MANUAL_SEED)

    # args for different backbones
    cfg.parse(args['resnet18'])
    run_id = random.randint(1, 100000)
    summary_dir = '/home/lzy/summary/generateDepth/' + 'train_depth_1e-3_0909_' + str(
        run_id)
    if not os.path.exists(summary_dir):
        os.mkdir(summary_dir)
    writer = SummaryWriter(summary_dir)
    cfg.LR = 0.001
    os.environ["CUDA_VISIBLE_DEVICES"] = '0,1,2'
    device_ids = torch.cuda.device_count()
    print('device_ids:', device_ids)
    # project_name = reduce(lambda x, y: str(x) + '/' + str(y), os.path.realpath(__file__).split(os.sep)[:-1])
    # util.mkdir('logs')
    # normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
    #                                 std=[0.229, 0.224, 0.225])
    # normalize=transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])
    # data
    train_dataset = dataset.SingleDataset(
        cfg,
        data_dir=cfg.DATA_DIR_TRAIN,
        transform=transforms.Compose([
            dataset.Resize((cfg.LOAD_SIZE, cfg.LOAD_SIZE)),
            dataset.RandomCrop((cfg.FINE_SIZE, cfg.FINE_SIZE)),
            dataset.RandomHorizontalFlip(),
            # dataset.RandomRotate(),
            # dataset.RandomFlip(),
            # dataset.PILBrightness(0.4),
            # dataset.PILContrast(0.4),
            # dataset.PILColorBalance(0.4),
            # dataset.PILSharpness(0.4),
            dataset.ToTensor(),
            dataset.Normalize(mean=[0.485, 0.456, 0.406],
                              std=[0.229, 0.224, 0.225])
        ]))

    val_dataset = dataset.SingleDataset(
        cfg,
        data_dir=cfg.DATA_DIR_VAL,
        transform=transforms.Compose([
            dataset.Resize((cfg.LOAD_SIZE, cfg.LOAD_SIZE)),
            dataset.CenterCrop((cfg.FINE_SIZE, cfg.FINE_SIZE)),
            dataset.ToTensor(),
            dataset.Normalize(mean=[0.485, 0.456, 0.406],
                              std=[0.229, 0.224, 0.225])
        ]))
    # train_loader = torch.utils.data.DataLoader(
    #     train_dataset, batch_size=cfg.BATCH_SIZE, shuffle=True,
    #     num_workers=4, pin_memory=True, sampler=None)
    # val_loader = torch.utils.data.DataLoader(
    #     val_dataset,batch_size=16, shuffle=False,
    #     num_workers=4, pin_memory=True)
    train_loader = DataProvider(cfg,
                                dataset=train_dataset,
                                batch_size=150,
                                shuffle=True)
    val_loader = DataProvider(cfg,
                              dataset=val_dataset,
                              batch_size=120,
                              shuffle=False)

    # class weights
    # num_classes_train = list(Counter([i[1] for i in train_loader.dataset.imgs]).values())
    # cfg.CLASS_WEIGHTS_TRAIN = torch.FloatTensor(num_classes_train)

    # writer = SummaryWriter(log_dir=cfg.LOG_PATH)  # tensorboard

    # net_classification_1=resnet50(pretrained=True)
    # net_classification_2=resnet50(pretrained=True)

    net_classification_2 = models.__dict__['resnet50'](num_classes=365)
    # net_classification_2=torchvision.models.resnext101_32x8d(pretrained=True, progress=True)
    # net_classification_2 = models.__dict__['resnet18'](num_classes=365)
    # net_classification_2=torch.hub.load('facebookresearch/WSL-Images', 'resnext101_32x16d_wsl')
    net_classification_2.fc = nn.Linear(2048, 19)

    # net_classification_2=torch.hub.load('facebookresearch/WSL-Images', 'resnext101_32x16d_wsl')
    # for param in net_classification_1.parameters():
    #     param.requires_grad = False
    # for param in net_classification_2.parameters():
    #     param.requires_grad = True
    # net_classification_1.fc = nn.Sequential(nn.Dropout(p=0.5),nn.Linear(2048, 1024),nn.LeakyReLU(inplace=True),nn.Linear(1024,67))
    # net_classification_2.fc = nn.Sequential(nn.Dropout(p=0.5),nn.Linear(2048,1024),nn.LeakyReLU(inplace=True),nn.Linear(1024,67))

    # net_classification_1.load_state_dict(torch.load("./bestmodel/best_model_resnext_16d_2048_1024_dropout_0.5_b.pkl"))、
    # net_classification_2.load_state_dict(torch.load("./bestmodel/best_model_resnext_16d_2048_1024_dropout_0.5_b.pkl"))
    # net_classification_2
    load_path = "resnet50_Depth_sunrgbd_best_0909_5e-4_.pth.tar"
    checkpoint = torch.load(load_path,
                            map_location=lambda storage, loc: storage)
    state_dict = {
        str.replace(k, 'module.', ''): v
        for k, v in checkpoint['state_dict'].items()
    }
    best_mean_depth = checkpoint['best_mean_2']
    print("load sunrgbd dataset:", best_mean_depth)
    net_classification_2.load_state_dict(state_dict)
    # print(net_classification_1)
    # num_ftrs = net_classification_1.fc.in_features
    # net_classification_1.fc = nn.Linear(num_ftrs, cfg.NUM_CLASSES)
    # num_ftrs = net_classification_2.fc.in_features
    # net_classification_2.fc = nn.Linear(num_ftrs, cfg.NUM_CLASSES)
    # net_classification_2.fc = nn.Sequential(nn.Dropout(p=0.5),nn.Linear(2048,1024),nn.LeakyReLU(inplace=True),nn.Linear(1024,67))
    net_classification_2.fc = nn.Linear(2048, 67)
    init_weights(net_classification_2.fc, 'normal')
    print(net_classification_2)

    # net_classification_1.cuda()
    net_classification_2.cuda()
    cudnn.benchmark = True

    if cfg.GENERATE_Depth_DATA:
        print('GENERATE_Depth_DATA model set')
        cfg_generate = copy.deepcopy(cfg)
        cfg_generate.CHECKPOINTS_DIR = '/home/lzy/generateDepth/bestmodel/before_best.pth'
        cfg_generate.GENERATE_Depth_DATA = False
        cfg_generate.NO_UPSAMPLE = False
        checkpoint = torch.load(cfg_generate.CHECKPOINTS_DIR)
        model = define_TrecgNet(cfg_generate, upsample=True, generate=True)
        load_checkpoint_depth(model,
                              cfg_generate.CHECKPOINTS_DIR,
                              checkpoint,
                              data_para=True)
        generate_model = torch.nn.DataParallel(model).cuda()
        generate_model.eval()

    # net_classification_1 = torch.nn.DataParallel(net_classification_1).cuda()
    net_classification_2 = torch.nn.DataParallel(net_classification_2).cuda()
    criterion = nn.CrossEntropyLoss().cuda()

    # best_mean_1=0
    best_mean_2 = 0
    # optimizer_2 = torch.optim.Adam(net_classification_2.parameters(), lr=cfg.LR, betas=(0.5, 0.999))
    optimizer_2 = torch.optim.SGD(net_classification_2.parameters(),
                                  lr=cfg.LR,
                                  momentum=cfg.MOMENTUM,
                                  weight_decay=cfg.WEIGHT_DECAY)

    schedulers = get_scheduler(optimizer_2, cfg, cfg.LR_POLICY)
    for epoch in range(0, 1000):
        if global_all_step > cfg.NITER_TOTAL:
            break
        # meanacc_1,meanacc_2=validate(val_loader, net_classification_1,net_classification_2,generate_model,criterion,epoch)
        # for param_group in optimizer_2.param_groups:
        #     lr_t = param_group['lr']
        #     print('/////////learning rate = %.7f' % lr_t)
        #     writer.add_scalar('LR',lr_t,global_step=epoch)
        printlr(optimizer_2, writer, epoch)

        train(train_loader, schedulers, net_classification_2, generate_model,
              criterion, optimizer_2, epoch, writer)
        meanacc_2 = validate(val_loader, net_classification_2, generate_model,
                             criterion, epoch, writer)
        # meanacc_2=validate(val_loader,net_classification_2,generate_model,criterion,epoch,writer)

        # train(train_loader,net_classification_2,generate_model,criterion,optimizer_2,epoch,writer)
        # meanacc_2=validate(val_loader,net_classification_2,generate_model,criterion,epoch,writer)

        # writer.add_image(depth_image[0])
        # save best
        # if meanacc_1>best_mean_1:
        #     best_mean_1=meanacc_1
        #     print('best_mean_color:',str(best_mean_1))
        #     save_checkpoint({
        #         'epoch': epoch,
        #         'arch': cfg.ARCH,
        #         'state_dict': net_classification_1.state_dict(),
        #         'best_mean_1': best_mean_1,
        #         'optimizer' : optimizer_1.state_dict(),
        #     },CorD=True)

        if meanacc_2 > best_mean_2:
            best_mean_2 = meanacc_2
            print('best_mean_depth:', str(best_mean_2))
            save_checkpoint(
                {
                    'epoch': epoch,
                    'arch': cfg.ARCH,
                    'state_dict': net_classification_2.state_dict(),
                    'best_mean_2': best_mean_2,
                    'optimizer': optimizer_2.state_dict(),
                },
                CorD=False)
        # print('best_mean_color:',str(best_mean_1))
        # writer.add_scalar('mean_acc_color', meanacc_1, global_step=epoch)
        writer.add_scalar('mean_acc_depth', meanacc_2, global_step=epoch)
        # writer.add_scalar('best_meanacc_color', best_mean_1, global_step=epoch)
        writer.add_scalar('best_meanacc_depth', best_mean_2, global_step=epoch)

    writer.close()
예제 #25
0
파일: trainer.py 프로젝트: altus88/striate
 def init_data_provider(self):
   self.train_dp = DataProvider(self.batch_size, self.data_dir, self.train_range)
   self.test_dp = DataProvider(self.batch_size, self.data_dir, self.test_range)
예제 #26
0
파일: trainer.py 프로젝트: altus88/striate
class Trainer:
  CHECKPOINT_REGEX = None
  def __init__(self, test_id, data_dir, checkpoint_dir, train_range, test_range, test_freq,
      save_freq, batch_size, num_epoch, image_size, image_color, learning_rate, n_out,
      autoInit=True, adjust_freq = 1, factor = 1.0):
    self.test_id = test_id
    self.data_dir = data_dir
    self.checkpoint_dir = checkpoint_dir
    self.train_range = train_range
    self.test_range = test_range
    self.test_freq = test_freq
    self.save_freq = save_freq
    self.batch_size = batch_size
    self.num_epoch = num_epoch
    self.image_size = image_size
    self.image_color = image_color
    self.learning_rate = learning_rate
    self.n_out = n_out
    self.factor = factor
    self.adjust_freq = adjust_freq
    self.regex = re.compile('^test%d-(\d+)\.(\d+)$' % self.test_id)

    self.init_data_provider()
    self.image_shape = (self.batch_size, self.image_color, self.image_size, self.image_size)
    self.train_outputs = []
    self.test_outputs = []
    self.net = FastNet(self.learning_rate, self.image_shape, self.n_out, autoAdd = autoInit)

    self.num_batch = self.curr_epoch = self.curr_batch = 0
    self.train_data = None
    self.test_data = None

    self.num_train_minibatch = 0
    self.num_test_minibatch = 0
    self.checkpoint_file = ''

  def init_data_provider(self):
    self.train_dp = DataProvider(self.batch_size, self.data_dir, self.train_range)
    self.test_dp = DataProvider(self.batch_size, self.data_dir, self.test_range)


  def get_next_minibatch(self, i, train = TRAIN):
    if train == TRAIN:
      num = self.num_train_minibatch
      data = self.train_data
    else:
      num = self.num_test_minibatch
      data = self.test_data

    batch_data = data['data']
    batch_label = data['labels']
    batch_size = self.batch_size

    if i == num -1:
      input = batch_data[:, i * batch_size: -1]
      label = batch_label[i* batch_size : -1]
    else:
      input = batch_data[:, i * batch_size: (i +1)* batch_size]
      label = batch_label[i * batch_size: (i + 1) * batch_size]

    return input, label


  def save_checkpoint(self):
    model = {}
    model['batchnum'] = self.train_dp.get_batch_num()
    model['epoch'] = self.num_epoch + 1
    model['layers'] = self.net.get_dumped_layers()
    model['train_outputs'] = self.train_outputs
    model['test_outputs'] = self.test_outputs

    dic = {'model_state': model, 'op':None}
    saved_filename = [f for f in os.listdir(self.checkpoint_dir) if self.regex.match(f)]
    for f in saved_filename:
      os.remove(os.path.join(self.checkpoint_dir, f))
    checkpoint_filename = "test%d-%d.%d" % (self.test_id, self.curr_epoch, self.curr_batch)
    checkpoint_file_path = os.path.join(self.checkpoint_dir, checkpoint_filename)
    self.checkpoint_file = checkpoint_file_path
    print checkpoint_file_path
    with open(checkpoint_file_path, 'w') as f:
      cPickle.dump(dic, f)

  def get_test_error(self):
    start = time.time()
    _, _, self.test_data = self.test_dp.get_next_batch()

    self.num_test_minibatch = ceil(self.test_data['data'].shape[1], self.batch_size)
    for i in range(self.num_test_minibatch):
      input, label = self.get_next_minibatch(i, TEST)
      label = np.array(label).astype(np.float32)
      label.shape = (label.size, 1)
      self.net.train_batch(input, label, TEST)
    cost , correct, numCase, = self.net.get_batch_information()
    self.test_outputs += [({'logprob': [cost, 1-correct]}, numCase, time.time() - start)]
    print 'error: %f logreg: %f time: %f' % (1-correct, cost, time.time() -
      start)

  def check_continue_trainning(self):
    return self.curr_epoch <= self.num_epoch

  def check_test_data(self):
    return self.num_batch % self.test_freq == 0

  def check_save_checkpoint(self):
    return self.num_batch % self.save_freq == 0

  def check_adjust_lr(self):
    return self.num_batch % self.adjust_freq == 0

  def train(self):
    self.curr_epoch, self.curr_batch, self.train_data = self.train_dp.get_next_batch()#self.train_dp.wait()
    while self.check_continue_trainning():
      start = time.time()
      self.num_train_minibatch = ceil(self.train_data['data'].shape[1], self.batch_size)

      for i in range(self.num_train_minibatch):
        input, label = self.get_next_minibatch(i)
        self.net.train_batch(input, label)

      cost , correct, numCase = self.net.get_batch_information()
      self.train_outputs += [({'logprob': [cost, 1-correct]}, numCase, time.time() - start)]
      print '%d.%d: error: %f logreg: %f time: %f' % (self.curr_epoch, self.curr_batch, 1-correct, cost, time.time() -
          start)

      self.num_batch += 1
      if self.check_test_data():
        print '---- test ----'
        self.get_test_error()
        print '------------'

      if self.factor != 1.0 and self.check_adjust_lr():
        print '---- adjust learning rate ----'
        self.net.adjust_learning_rate(self.factor)
        print '--------'

      if self.check_save_checkpoint():
        print '---- save checkpoint ----'
        self.save_checkpoint()
        print '------------'

      self.curr_epoch, self.curr_batch, self.train_data = self.train_dp.get_next_batch()##self.train_dp.wait()

    if self.num_batch % self.save_freq != 0:
      print '---- save checkpoint ----'
      self.save_checkpoint()
예제 #27
0
def build_encoder(args):
    train_fn = train_vae.train_vae
    device = 'cuda' if torch.cuda.is_available() else 'cpu'
    with open(os.path.join('model_save', 'train_params.json'), 'r') as f:
        training_params = json.load(f)

    parsed_ft_params = parsing_utils.parse_hyper_vae_ft_evaluation_result(
        metric_name=args.metric)
    training_params['unlabeled'].update(parsed_ft_params[0])
    training_params['labeled']['train_num_epochs'] = parsed_ft_params[1]

    training_params.update({
        'device':
        device,
        'model_save_folder':
        os.path.join('model_save', 'ae500'),
        'es_flag':
        False,
        'retrain_flag':
        True
    })

    task_save_folder = training_params['model_save_folder']
    safe_make_dir(training_params['model_save_folder'])

    random.seed(2020)

    data_provider = DataProvider(
        batch_size=training_params['unlabeled']['batch_size'],
        target=args.measurement)
    training_params.update({
        'input_dim': data_provider.shape_dict['gex'],
        'output_dim': data_provider.shape_dict['target']
    })
    encoder, historys = train_fn(
        dataloader=data_provider.get_unlabeled_gex_dataloader(),
        **wrap_training_params(training_params, type='unlabeled'))

    with open(
            os.path.join(training_params['model_save_folder'],
                         f'unlabel_train_history.pickle'), 'wb') as f:
        for history in historys:
            pickle.dump(dict(history), f)

    ft_evaluation_metrics = defaultdict(list)
    labeled_dataloader = data_provider.get_labeled_gex_dataloader()
    ft_encoder = deepcopy(encoder)
    target_regressor, ft_historys = fine_tuning.fine_tune_encoder_new(
        encoder=ft_encoder,
        train_dataloader=labeled_dataloader,
        val_dataloader=labeled_dataloader,
        test_dataloader=None,
        seed=2021,
        metric_name=args.metric,
        task_save_folder=task_save_folder,
        **wrap_training_params(training_params, type='labeled'))
    for metric in ['dpearsonr', 'drmse', 'cpearsonr', 'crmse']:
        try:
            ft_evaluation_metrics[metric].append(ft_historys[-2][metric][-1])
        except:
            pass

    with open(os.path.join(task_save_folder, f'ft_evaluation_results.json'),
              'w') as f:
        json.dump(ft_evaluation_metrics, f)

    torch.save(target_regressor.encoder.state_dict(),
               os.path.join('model_save', 'reference_encoder.pt'))
예제 #28
0
        cfg,
        data_dir=cfg.DATA_DIR_TRAIN,
        transform=transforms.Compose(train_transforms))

    val_dataset = SPL10.SPL10_Dataset(
        cfg,
        data_dir=cfg.DATA_DIR_VAL,
        transform=transforms.Compose([
            SPL10.Resize((cfg.LOAD_SIZE, cfg.LOAD_SIZE)),
            SPL10.CenterCrop((cfg.FINE_SIZE, cfg.FINE_SIZE)),
            RGB2Lab(),
            SPL10.ToTensor(),
            SPL10.Normalize(mean=[0.485, 0.456, 0.406],
                            std=[0.229, 0.224, 0.225])
        ]))
    train_loader = DataProvider(cfg, dataset=train_dataset)
    val_loader = DataProvider(cfg, dataset=val_dataset, shuffle=False)

    num_classes_train = list(
        Counter([i[1] for i in train_loader.dataset.imgs]).values())
    cfg.CLASS_WEIGHTS_TRAIN = torch.FloatTensor(num_classes_train)

    model = Contrastive_CrossModal_Conc(cfg, device=device)
    model = nn.DataParallel(model).to(device)
    optim = Adam(model.parameters(), lr=cfg.LR)
    load_model = False
    if load_model:
        model = torch.load('./checkpoint/model_1_LAB.mdl')
        print("load pretrained model")
    # loss_optim = Adam(infomax_fn.parameters(), lr=2e-4)
    # cls_criterion = torch.nn.CrossEntropyLoss(cfg.CLASS_WEIGHTS_TRAIN.to(device))
예제 #29
0
파일: trainer.py 프로젝트: wqren/striate
 def init_subnet_data_provider(self):
   dp = DataProvider.get_by_name('intermediate')
   count = self.train_dumper.get_count()
   self.train_dp = dp(self.train_output_filename,  range(0, count), 'fc')
   count = self.test_dumper.get_count()
   self.test_dp = dp(self.test_output_filename, range(0, count), 'fc')
예제 #30
0
파일: trainer.py 프로젝트: wqren/striate
 def set_category_range(self, r):
   dp = DataProvider.get_by_name(self.data_provider)
   self.train_dp = dp(self.data_dir, self.train_range, category_range = range(r))
   self.test_dp = dp(self.data_dir, self.test_range, category_range = range(r))
예제 #31
0
from config import Config
from data import DataProvider
from gensim.models.word2vec import Word2Vec
import numpy as np
import os

flag = "tag"
conf = Config(flag, "tag" , 300)

if not os.path.exists(conf.path_word_w2c) and not os.path.exists(conf.path_doc_w2c):
    doc_embed = np.load(conf.path_doc_npy + ".npy")[0]
    dp = DataProvider(conf)

    # generate doc embedding file
    f = open(conf.path_doc_w2c,"w")
    f.write(str(len(dp.idx2prod)))
    f.write(" ")
    f.write(str(conf.dim_item))
    f.write("\n")
    idx = 0
    batch = ""
    for word in dp.idx2prod:
        batch = "".join([batch, word])
        batch = "".join([batch, " "])

        for i in range(conf.dim_item):
            batch = "".join([batch, str(doc_embed[idx][i])])
            batch = "".join([batch, " "])

        batch = "".join([batch, "\n"])
        idx += 1
예제 #32
0
def main():
    global cfg
    cfg = DefaultConfig()
    args = {
        'resnet18': RESNET18_SUNRGBD_CONFIG().args(),
    }

    # args for different backbones
    cfg.parse(args['resnet18'])
    cfg.LR = 1e-4
    cfg.EPOCHS = 200
    # print('cfg.EPOCHS:',cfg.EPOCHS)
    os.environ["CUDA_VISIBLE_DEVICES"] = '0,1'

    # normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
    #                                 std=[0.229, 0.224, 0.225])
    # dataset.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])
    # data
    train_dataset = dataset.SingleDataset(
        cfg,
        data_dir=cfg.DATA_DIR_TRAIN,
        transform=transforms.Compose([
            dataset.Resize((cfg.LOAD_SIZE, cfg.LOAD_SIZE)),
            dataset.RandomCrop((cfg.FINE_SIZE, cfg.FINE_SIZE)),
            dataset.RandomHorizontalFlip(),
            dataset.ToTensor(),
            dataset.Normalize(mean=[0.485, 0.456, 0.406],
                              std=[0.229, 0.224, 0.225])
        ]))

    val_dataset = dataset.SingleDataset(
        cfg,
        data_dir=cfg.DATA_DIR_VAL,
        transform=transforms.Compose([
            dataset.Resize((cfg.LOAD_SIZE, cfg.LOAD_SIZE)),
            dataset.CenterCrop((cfg.FINE_SIZE, cfg.FINE_SIZE)),
            dataset.ToTensor(),
            dataset.Normalize(mean=[0.485, 0.456, 0.406],
                              std=[0.229, 0.224, 0.225])
        ]))
    # train_loader = torch.utils.data.DataLoader(
    #     train_dataset, batch_size=cfg.BATCH_SIZE, shuffle=True,
    #     num_workers=4, pin_memory=True, sampler=None)
    # val_loader = torch.utils.data.DataLoader(
    #     val_dataset,batch_size=cfg.BATCH_SIZE, shuffle=False,
    #     num_workers=4, pin_memory=True)
    train_loader = DataProvider(cfg,
                                dataset=train_dataset,
                                batch_size=20,
                                shuffle=True)
    val_loader = DataProvider(cfg,
                              dataset=val_dataset,
                              batch_size=5,
                              shuffle=False)

    run_id = random.randint(1, 100000)
    summary_dir = '/home/lzy/summary/generateDepth/' + 'finetuning_nofix_' + str(
        run_id)
    if not os.path.exists(summary_dir):
        os.mkdir(summary_dir)
    writer = SummaryWriter(summary_dir)

    if cfg.GENERATE_Depth_DATA:
        print('GENERATE_Depth_DATA model set')
        cfg_generate = copy.deepcopy(cfg)

        cfg_generate.CHECKPOINTS_DIR = '/home/lzy/generateDepth/checkpoints/best_AtoB/trecg_AtoB_best.pth'
        cfg_generate.GENERATE_Depth_DATA = False
        cfg_generate.NO_UPSAMPLE = False
        checkpoint = torch.load(cfg_generate.CHECKPOINTS_DIR)
        model = define_TrecgNet(cfg_generate, upsample=True, generate=True)
        load_checkpoint_depth(model,
                              cfg_generate.CHECKPOINTS_DIR,
                              checkpoint,
                              data_para=True)
        generate_model = torch.nn.DataParallel(model).cuda()
        generate_model.eval()

    model = ReD_Model(cfg)
    policies = model.get_optim_policies()
    model = torch.nn.DataParallel(model).cuda()
    criterion = nn.CrossEntropyLoss().cuda()
    best_mean = 0
    # optimizer= torch.optim.Adam(policies, cfg.LR,
    #                         weight_decay=cfg.WEIGHT_DECAY)
    optimizer = torch.optim.SGD(policies,
                                cfg.LR,
                                momentum=cfg.MOMENTUM,
                                weight_decay=cfg.WEIGHT_DECAY)

    for epoch in range(cfg.START_EPOCH, cfg.EPOCHS + 1):
        adjust_learning_rate(optimizer, epoch)
        # mean_acc=validate(val_loader,model,criterion,generate_model,epoch,writer)

        train(train_loader, model, criterion, generate_model, optimizer, epoch,
              writer)
        mean_acc = validate(val_loader, model, criterion, generate_model,
                            epoch, writer)
        if mean_acc > best_mean:
            best_mean = mean_acc
            print('best mean accuracy:', best_mean)
        else:
            print('best mean accuracy:', best_mean)
        writer.add_scalar('mean_acc_color', mean_acc, global_step=epoch)
        writer.add_scalar('best_meanacc_color', best_mean, global_step=epoch)
    writer.close()
예제 #33
0
파일: trainer.py 프로젝트: phecy/striate
 def init_data_provider(self):
   dp = DataProvider.get_by_name(self.data_provider)
   self.train_dp = dp(self.data_dir, self.train_range)
   self.test_dp = dp(self.data_dir, self.test_range)
예제 #34
0
파일: trainer.py 프로젝트: wqren/striate
 def init_subnet_data_provider(self):
     dp = DataProvider.get_by_name('intermediate')
     count = self.train_dumper.get_count()
     self.train_dp = dp(self.train_output_filename, range(0, count), 'fc')
     count = self.test_dumper.get_count()
     self.test_dp = dp(self.test_output_filename, range(0, count), 'fc')
예제 #35
0
파일: app.py 프로젝트: xander27/krasn_gen
    if os.path.exists(_CUR_LOG_FILE_NAME):
        os.rename(_CUR_LOG_FILE_NAME, _PRED_LOG_FILE_NAME)
    log.basicConfig( level=log.INFO)
#    log.basicConfig(filename=_CUR_LOG_FILE_NAME, level=log.INFO)
    log.info('start')

if __name__ == '__main__':
#    try:
        FILE_NAME = "data.dat"
        INIT_DATA_FILE_NAME = 'init.dat'
        TEST_FILE_NAME = "test_data.dat"
        TEST_INIT_DATA_FILE_NAME = 'test_init.dat'
#        binder.bind(str, annotated_with="data_file_name", to_instance = FILE_NAME)
#        binder.bind(str, annotated_with="init_file_name", to_instance = INIT_DATA_FILE_NAME)
        _init_log()
        dataProvider = DataProvider(FILE_NAME, INIT_DATA_FILE_NAME)
#        test_data = DataProvider(TEST_FILE_NAME, TEST_INIT_DATA_FILE_NAME).get_data()
#        assert(calc_class_re())
        app = QApplication(sys.argv)
        data = dataProvider.get_data()
#        print data
        classifier = c45(data, max_repeat_var=10)
        form = MainWindow(data, classifier)

##        print data
#        classifier = c45(data, max_repeat_var=10)
#        pos_sum = 0
#        for row, target in zip(data.data, data.target):
#            pos = 0
#            for l, c in classifier.get_labels_count(row).items():
#                pos += 1
예제 #36
0
파일: trainer.py 프로젝트: wqren/striate
 def set_num_group(self, n):
     dp = DataProvider.get_by_name(self.data_provider)
     self.train_dp = dp(self.data_dir, self.train_range, n)
     self.test_dp = dp(self.data_dir, self.test_range, n)
                      os.path.realpath(__file__).split(os.sep)[:-1])
util.mkdir('logs')

val_dataset = dataset.AlignedConcDataset(
    cfg,
    data_dir=cfg.DATA_DIR_VAL,
    transform=transforms.Compose([
        dataset.Resize((cfg.LOAD_SIZE, cfg.LOAD_SIZE)),
        dataset.CenterCrop((cfg.FINE_SIZE, cfg.FINE_SIZE)),
        dataset.ToTensor(),
        dataset.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])
    ]))
batch_size_val = cfg.BATCH_SIZE

val_loader = DataProvider(cfg,
                          dataset=val_dataset,
                          batch_size=batch_size_val,
                          shuffle=False)
writer = SummaryWriter(log_dir=cfg.LOG_PATH)  # tensorboard
model = TRecgNet_Upsample_Resiual(cfg, writer)
model.set_data_loader(None, val_loader, None)
model.net = nn.DataParallel(model.net).to(model.device)
model.set_log_data(cfg)


def evaluate():

    checkpoint_path = os.path.join(cfg.CHECKPOINTS_DIR, cfg.RESUME_PATH)
    checkpoint = torch.load(checkpoint_path)
    load_epoch = checkpoint['epoch']
    model.load_checkpoint(model.net,
                          checkpoint_path,
예제 #38
0
    logger.info('read text')
    train_text, test_text = get_text()

    args = parse_args()

    seq_length = args.seq_length
    batch_size = args.batch_size
    rnn_size = args.rnn_size
    context_size = args.context_size
    num_epochs = args.num_epoch
    learning_rate = args.lr
    alpha = args.alpha

    logger.info('build vocabulary')
    data_provider = DataProvider(train_text, seq_length, batch_size, logger)
    X_train, y_train = data_provider.get_data()

    logger.info('X.shape={}, y.shape={}'.format(X_train.shape, y_train.shape))

    vocab_size = data_provider.vocab_size

    input_data = tf.placeholder(tf.float32,
                                [batch_size, seq_length, vocab_size])
    targets = tf.placeholder(tf.float32, [batch_size, vocab_size])

    test_data_provider = DataProvider(test_text, seq_length, batch_size,
                                      logger, data_provider.vocab)

    if args.cell == 'lstm':
        cell = LSTMCell(num_units=rnn_size)
예제 #39
0
    # Model
    net = Net()
    net.push(Conv2d(5, 5, 1, 6))  # 1x28x28 -> 6x24x24
    net.push(Relu())
    net.push(Maxpooling(2, 2))  # 6x24x24 -> 6x12x12
    net.push(Conv2d(5, 5, 6, 16))  # 6x12x12 -> 16x8x8
    net.push(Relu())
    net.push(Maxpooling(2, 2))  # 16x8x8 -> 16x4x4
    net.push(Reshape((256)))
    net.push(Linear(256, 84))
    net.push(Relu())
    net.push(Softmax(84, 10))

    # Data
    data = DataProvider()
    n = 10000
    data.train_input(x[:n], y[:n])
    data.test_input(xt, yt)
    data.batch_size(16)

    lr = 0.0009
    gamma = 0.9
    for epoch in xrange(50):
        print 'Epoch: ', epoch

        # Training (Mini-batch)
        now = time.time()
        for _ in xrange(data.batch_run()):
            net.input(data.next_batch())
            net.forward()
예제 #40
0
from config import Config
from data import DataProvider
from gensim.models.word2vec import Word2Vec
import numpy as np
import os

flag = "tag"
conf = Config(flag, "tag", 300)

if not os.path.exists(conf.path_word_w2c) and not os.path.exists(
        conf.path_doc_w2c):
    doc_embed = np.load(conf.path_doc_npy + ".npy")[0]
    dp = DataProvider(conf)

    # generate doc embedding file
    f = open(conf.path_doc_w2c, "w")
    f.write(str(len(dp.idx2prod)))
    f.write(" ")
    f.write(str(conf.dim_item))
    f.write("\n")
    idx = 0
    batch = ""
    for word in dp.idx2prod:
        batch = "".join([batch, word])
        batch = "".join([batch, " "])

        for i in range(conf.dim_item):
            batch = "".join([batch, str(doc_embed[idx][i])])
            batch = "".join([batch, " "])

        batch = "".join([batch, "\n"])
예제 #41
0
os.environ['THEANO_FLAGS'] = 'device=gpu'

import os
os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"   # see issue #152
os.environ["CUDA_VISIBLE_DEVICES"]="1"

config = tf.ConfigProto(log_device_placement=True, allow_soft_placement=True)
config.gpu_options.allow_growth = True
session = tf.Session(config=config)
K.set_session(session)

conf = Config(flag, args[2], int(args[3]))
print(flag)

# get data
dp = DataProvider(conf)
n_terms = len(dp.idx2word)
word_embed_data = np.array(dp.word_embed)

item_embed_data = np.random.rand(dp.get_item_size(), conf.dim_word)
print("finish data processing")

# define model
word_input = Input(shape=(1,), dtype ="int32", name ="word_idx")
item_pos_input = Input(shape=(1,), dtype ="int32", name ="item_pos_idx")
item_neg_input = Input(shape=(1,), dtype ="int32", name ="item_neg_idx")

word_embed = Embedding(output_dim=conf.dim_word, input_dim=n_terms, input_length=1, name="word_embed",
                       weights=[word_embed_data], trainable=False)
item_embed = Embedding(output_dim=conf.dim_word, input_dim=dp.get_item_size(), input_length=1, name="item_embed",
                       weights=[item_embed_data], trainable=True)
예제 #42
0
class Net():
    def __init__(self,
                 net,
                 snapshot_prefix,
                 dp_params,
                 preproc,
                 iter=None,
                 test=False):
        if iter is not None:
            self.N = caffe.Net(net,
                               snapshot_prefix + str(iter) + '.caffemodel',
                               caffe.TRAIN)
            self.iter = iter
        else:
            if test:
                self.N = caffe.Net(net, snapshot_prefix, caffe.TEST)
#                self.N = caffe.Net(net, caffe.TRAIN)
            else:
                self.N = caffe.Net(net, snapshot_prefix, caffe.TRAIN)
            self.iter = 0

        # Data provider
        self.dp = DataProvider(dp_params, preproc)
        self.bsize = self.dp.batch_size

        self.prevs = {}
        self.test = test

    def forward(self):
        ind = self.iter * self.bsize
        if self.test:
            _data, _labels = self.dp.get_batch_test(ind)
        else:
            _data, _labels = self.dp.get_batch(ind)

        # set data as input
        self.N.blobs['data'].data[...] = _data
        for label_key in _labels.keys():
            self.N.blobs[label_key].data[...] = _labels[label_key].reshape(
                self.N.blobs[label_key].data.shape)

        # Forward
#        t0 = time.time()
        out = self.N.forward()
        self.iter += 1
        return out

    def backward(self):
        self.N.backward()
        # update filter parameters
        #        t0 = time.time()
        for layer_name, lay in zip(self.N._layer_names, self.N.layers):
            for blobind, blob in enumerate(lay.blobs):
                diff = blob.diff[:]
                key = (layer_name, blobind)
                if key in self.prevs:
                    previous_change = self.prevs[key]
                else:
                    previous_change = 0

                lr = 0.01
                wd = 0.0005
                momentum = 0.9
                if blobind == 1:
                    lr = 2 * lr
                    wd = 0
                if lay.type == "BatchNorm":
                    lr = 0
                    wd = 0
                change = momentum * previous_change - lr * diff - lr * wd * blob.data[:]

                blob.data[:] += change
                self.prevs[key] = change

    def empty_diff(self):
        for layer_name, lay in zip(self.N._layer_names, self.N.layers):
            for blobind, blob in enumerate(lay.blobs):
                blob.diff[:] = 0
예제 #43
0
def main():
    global cfg
    cfg = DefaultConfig()
    args = {
        'resnet18': RESNET18_SUNRGBD_CONFIG().args(),
    }

    # Setting random seed
    # if cfg.MANUAL_SEED is None:
    #     cfg.MANUAL_SEED = random.randint(1, 10000)
    # random.seed(cfg.MANUAL_SEED)
    # torch.manual_seed(cfg.MANUAL_SEED)

    # args for different backbones
    cfg.parse(args['resnet18'])
    run_id = random.randint(1, 100000)
    summary_dir = '/home/lzy/summary/generateDepth/' + 'train_rgb_' + str(
        run_id)
    if not os.path.exists(summary_dir):
        os.mkdir(summary_dir)
    writer = SummaryWriter(summary_dir)
    cfg.LR = 0.0001
    os.environ["CUDA_VISIBLE_DEVICES"] = '1,2'
    device_ids = torch.cuda.device_count()
    print('device_ids:', device_ids)
    # project_name = reduce(lambda x, y: str(x) + '/' + str(y), os.path.realpath(__file__).split(os.sep)[:-1])
    # util.mkdir('logs')
    # normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
    #                                 std=[0.229, 0.224, 0.225])
    # normalize=transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])
    # data
    train_dataset = dataset.SingleDataset(
        cfg,
        data_dir=cfg.DATA_DIR_TRAIN,
        transform=transforms.Compose([
            dataset.Resize((cfg.LOAD_SIZE, cfg.LOAD_SIZE)),
            dataset.RandomCrop((cfg.FINE_SIZE, cfg.FINE_SIZE)),
            dataset.RandomHorizontalFlip(),
            dataset.ToTensor(),
            dataset.Normalize(mean=[0.485, 0.456, 0.406],
                              std=[0.229, 0.224, 0.225])
        ]))

    val_dataset = dataset.SingleDataset(
        cfg,
        data_dir=cfg.DATA_DIR_VAL,
        transform=transforms.Compose([
            dataset.Resize((cfg.LOAD_SIZE, cfg.LOAD_SIZE)),
            dataset.CenterCrop((cfg.FINE_SIZE, cfg.FINE_SIZE)),
            dataset.ToTensor(),
            dataset.Normalize(mean=[0.485, 0.456, 0.406],
                              std=[0.229, 0.224, 0.225])
        ]))
    # train_loader = torch.utils.data.DataLoader(
    #     train_dataset, batch_size=cfg.BATCH_SIZE, shuffle=True,
    #     num_workers=4, pin_memory=True, sampler=None)
    # val_loader = torch.utils.data.DataLoader(
    #     val_dataset,batch_size=16, shuffle=False,
    #     num_workers=4, pin_memory=True)
    train_loader = DataProvider(cfg,
                                dataset=train_dataset,
                                batch_size=40,
                                shuffle=True)
    val_loader = DataProvider(cfg,
                              dataset=val_dataset,
                              batch_size=10,
                              shuffle=False)

    # class weights
    # num_classes_train = list(Counter([i[1] for i in train_loader.dataset.imgs]).values())
    # cfg.CLASS_WEIGHTS_TRAIN = torch.FloatTensor(num_classes_train)

    # writer = SummaryWriter(log_dir=cfg.LOG_PATH)  # tensorboard

    # net_classification_1=resnet50(pretrained=True)
    # net_classification_2=resnet50(pretrained=True)

    # net_classification_1 = models.__dict__['resnet18'](num_classes=365)
    # net_classification_2 = models.__dict__['resnet18'](num_classes=365)
    net_classification_1 = torch.hub.load('facebookresearch/WSL-Images',
                                          'resnext101_32x16d_wsl')
    # net_classification_2=torch.hub.load('facebookresearch/WSL-Images', 'resnext101_32x16d_wsl')
    # for param in net_classification_1.parameters():
    #     param.requires_grad = False
    # for param in net_classification_2.parameters():
    #     param.requires_grad = True
    net_classification_1.fc = nn.Sequential(nn.Dropout(p=0.5),
                                            nn.Linear(2048, 1024),
                                            nn.LeakyReLU(inplace=True),
                                            nn.Linear(1024, 67))
    # net_classification_2.fc = nn.Sequential(nn.Dropout(p=0.5),nn.Linear(2048, 1024),nn.LeakyReLU(inplace=True),nn.Linear(1024,67))

    net_classification_1.load_state_dict(
        torch.load(
            "./bestmodel/best_model_resnext_16d_2048_1024_dropout_0.5_b.pkl"))
    # net_classification_2.load_state_dict(torch.load("./bestmodel/best_model_resnext_16d_2048_1024_dropout_0.5_b.pkl"))
    # net_classification_2
    # load_path = "/home/dudapeng/workspace/pretrained/place/resnet18_places365.pth"
    # checkpoint = torch.load(load_path, map_location=lambda storage, loc: storage)
    # state_dict = {str.replace(k, 'module.', ''): v for k, v in checkpoint['state_dict'].items()}
    # net_classification_1.load_state_dict(state_dict)
    # net_classification_2.load_state_dict(state_dict)
    print(net_classification_1)

    # num_ftrs = net_classification_1.fc.in_features
    # net_classification_1.fc = nn.Linear(num_ftrs, cfg.NUM_CLASSES)
    # num_ftrs = net_classification_2.fc.in_features
    # net_classification_2.fc = nn.Linear(num_ftrs, cfg.NUM_CLASSES)

    net_classification_1.cuda()
    # net_classification_2.cuda()
    cudnn.benchmark = True

    # if cfg.GENERATE_Depth_DATA:
    #     print('GENERATE_Depth_DATA model set')
    #     cfg_generate = copy.deepcopy(cfg)
    #     cfg_generate.CHECKPOINTS_DIR='/home/lzy/generateDepth/checkpoints/best_AtoB/trecg_AtoB_best.pth'
    #     cfg_generate.GENERATE_Depth_DATA = False
    #     cfg_generate.NO_UPSAMPLE = False
    #     checkpoint = torch.load(cfg_generate.CHECKPOINTS_DIR)
    #     model = define_TrecgNet(cfg_generate, upsample=True,generate=True)
    #     load_checkpoint_depth(model,cfg_generate.CHECKPOINTS_DIR, checkpoint, data_para=True)
    #     generate_model = torch.nn.DataParallel(model).cuda()
    #     generate_model.eval()

    net_classification_1 = torch.nn.DataParallel(net_classification_1).cuda()
    # net_classification_2 = torch.nn.DataParallel(net_classification_2).cuda()
    criterion = nn.CrossEntropyLoss().cuda()

    best_mean_1 = 0
    # best_mean_2=0
    # optimizer = optim.SGD(model_ft.parameters(), lr=0.05,momentum=0.9)#,weight_decay=0.00005)
    optimizer_1 = torch.optim.SGD(net_classification_1.parameters(),
                                  lr=cfg.LR,
                                  momentum=cfg.MOMENTUM,
                                  weight_decay=cfg.WEIGHT_DECAY)
    # optimizer_2 = torch.optim.SGD(net_classification_2.parameters(),lr=cfg.LR,momentum=cfg.MOMENTUM,weight_decay=cfg.WEIGHT_DECAY)
    for epoch in range(0, 100):
        adjust_learning_rate(optimizer_1, epoch)
        # meanacc_1,meanacc_2=validate(val_loader, net_classification_1,net_classification_2,generate_model,criterion,epoch)
        #
        train(train_loader, net_classification_1, criterion, optimizer_1,
              epoch, writer)
        meanacc_1 = validate(val_loader, net_classification_1, criterion,
                             epoch, writer)
        # meanacc_2=validate(val_loader,net_classification_2,generate_model,criterion,epoch,writer)

        # train(train_loader,net_classification_2,generate_model,criterion,optimizer_2,epoch,writer)
        # meanacc_2=validate(val_loader,net_classification_2,generate_model,criterion,epoch,writer)

        # writer.add_image(depth_image[0])
        # save best
        if meanacc_1 > best_mean_1:
            best_mean_1 = meanacc_1
            print('best_mean_color:', str(best_mean_1))
            save_checkpoint(
                {
                    'epoch': epoch,
                    'arch': cfg.ARCH,
                    'state_dict': net_classification_1.state_dict(),
                    'best_mean_1': best_mean_1,
                    'optimizer': optimizer_1.state_dict(),
                },
                CorD=True)

        # if meanacc_2>best_mean_2:
        #     best_mean_2=meanacc_2
        #     print('best_mean_depth:',str(best_mean_2))
        #     save_checkpoint({
        #         'epoch': epoch,
        #         'arch': cfg.ARCH,
        #         'state_dict': net_classification_2.state_dict(),
        #         'best_mean_2': best_mean_2,
        #         'optimizer' : optimizer_2.state_dict(),
        #     },CorD=False)
        print('best_mean_color:', str(best_mean_1))
        writer.add_scalar('mean_acc_color', meanacc_1, global_step=epoch)
        # writer.add_scalar('mean_acc_depth', meanacc_2, global_step=epoch)
        writer.add_scalar('best_meanacc_color', best_mean_1, global_step=epoch)
        # writer.add_scalar('best_meanacc_depth', best_mean_2, global_step=epoch)

    writer.close()
예제 #44
0
os.environ['THEANO_FLAGS'] = 'device=gpu'

import os
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"  # see issue #152
os.environ["CUDA_VISIBLE_DEVICES"] = "3"

config = tf.ConfigProto(log_device_placement=False, allow_soft_placement=True)
config.gpu_options.allow_growth = True
session = tf.Session(config=config)
K.set_session(session)

conf = Config(flag, args[2], int(args[3]))
print(flag)

# get data
dp = DataProvider(conf)
n_terms = len(dp.idx2word)
word_embed_data = np.array(dp.word_embed)

item_embed_data = np.random.rand(dp.get_item_size(), conf.dim_item)
word_transfer_W = np.random.rand(conf.dim_word, conf.dim_item)
word_transfer_b = np.random.rand(conf.dim_item)
print("finish data processing")

# define model
word_input = Input(shape=(1, ), dtype="int32", name="word_idx")
item_pos_input = Input(shape=(1, ), dtype="int32", name="item_pos_idx")
item_neg_input = Input(shape=(1, ), dtype="int32", name="item_neg_idx")

word_embed = Embedding(output_dim=conf.dim_word,
                       input_dim=n_terms,
예제 #45
0
파일: train.py 프로젝트: piggypiggy/DCGAN
def train(args):
    height, width, channel = 28, 28, 1
    batch_size = args.batch_size
    z_size = args.nd  # 噪声维数
    real_img = tf.placeholder(tf.float32, [batch_size, height, width, channel],
                              name='img')
    z = tf.placeholder(tf.float32, [batch_size, z_size], name='z')
    label = tf.placeholder(tf.float32, [batch_size, 10], name='label')  # 0~9

    gan = model.GAN(height, width, channel)
    gan.set_batch_size(batch_size)
    fake_img = gan.generator(z, label)
    real_result = gan.discriminator(real_img, label, reuse=False)
    fake_result = gan.discriminator(fake_img, label, reuse=True)
    real = tf.reduce_sum(label * real_result, 1)
    fake = tf.reduce_sum(label * fake_result, 1)
    d_loss = -tf.reduce_mean(tf.log(real) + tf.log(1. - fake))
    g_loss = -tf.reduce_mean(tf.log(fake))

    t_vars = tf.trainable_variables()
    d_vars = [var for var in t_vars if 'dis' in var.name]
    g_vars = [var for var in t_vars if 'gen' in var.name]
    d_optimizer = tf.train.AdamOptimizer(learning_rate=0.0002, beta1=0.5) \
              .minimize(d_loss, var_list=d_vars)
    g_optimizer = tf.train.AdamOptimizer(learning_rate=0.0002, beta1=0.5) \
              .minimize(g_loss, var_list=g_vars)

    data = DataProvider()
    train_num = data.get_train_num()
    batch_num = int(train_num / args.batch_size)

    saver = tf.train.Saver(max_to_keep=1)
    model_dir = args.model_dir
    if (not os.path.exists(model_dir)):
        os.mkdir(model_dir)

    accuracy_real = tf.reduce_mean(
        tf.cast(tf.equal(tf.argmax(label, 1), tf.argmax(real_result, 1)),
                'float'))
    accuracy_fake = tf.reduce_mean(
        tf.cast(tf.equal(tf.argmax(label, 1), tf.argmax(fake_result, 1)),
                'float'))

    with tf.Session() as sess:
        counter = 0
        sess.run(tf.global_variables_initializer())
        for epoch in range(args.epoch):
            for batch in range(batch_num):
                counter += 1
                train_data, label_data = data.next_batch(batch_size)
                batch_z = np.random.normal(0, 1, [batch_size, z_size]).astype(
                    np.float_)

                sess.run(d_optimizer,
                         feed_dict={
                             real_img: train_data,
                             z: batch_z,
                             label: label_data
                         })
                sess.run(g_optimizer,
                         feed_dict={
                             z: batch_z,
                             label: label_data
                         })

                if (counter % 20 == 0):
                    dloss, gloss, ac_real, ac_fake = sess.run(
                        [d_loss, g_loss, accuracy_real, accuracy_fake],
                        feed_dict={
                            real_img: train_data,
                            z: batch_z,
                            label: label_data
                        })
                    print('iter:', counter, 'd_loss:', dloss, 'g_loss:', gloss,
                          'ac_real:', ac_real, 'ac_fake:', ac_fake)
                if (counter % 200 == 0):
                    saver.save(sess, os.path.join(model_dir, 'model'))