def __init__(self,
net,
snapshot_prefix,
dp_params,
preproc,
iter=None,
test=False):
if iter is not None:
self.N = caffe.Net(net,
snapshot_prefix + str(iter) + '.caffemodel',
caffe.TRAIN)
self.iter = iter
else:
if test:
self.N = caffe.Net(net, snapshot_prefix, caffe.TEST)
# self.N = caffe.Net(net, caffe.TRAIN)
else:
self.N = caffe.Net(net, snapshot_prefix, caffe.TRAIN)
self.iter = 0
# Data provider
self.dp = DataProvider(dp_params, preproc)
self.bsize = self.dp.batch_size
self.prevs = {}
self.test = test
def init_data_providers(self, dp_type=None):
if dp_type is not None:
self.dp_type = dp_type
self.train_batch_range = self.test_batch_range
self.dp_params['convnet'] = self
self.dp_params['img_size'] = self.img_size
try:
self.test_data_provider = DataProvider.get_instance(
self.data_path,
self.test_batch_range,
type=self.dp_type,
dp_params=self.dp_params,
test=True)
self.train_data_provider = DataProvider.get_instance(
self.data_path,
self.train_batch_range,
self.model_state["epoch"],
self.model_state["batchnum"],
type=self.dp_type,
dp_params=self.dp_params,
test=False)
except DataProviderException, e:
print "Unable to create data provider: %s" % e
self.print_data_providers()
sys.exit()
def main(args):
if args.method == 'dann':
train_fn = train_dann.train_dann
elif args.method == 'adda':
train_fn = train_adda.train_adda
elif args.method == 'dcc':
train_fn = train_dcc.train_dcc
elif args.method == 'cleitcs':
train_fn = train_cleitcs.train_cleitcs
else:
train_fn = train_coral.train_coral
device = 'cuda' if torch.cuda.is_available() else 'cpu'
with open(os.path.join('model_save', 'train_params.json'), 'r') as f:
training_params = json.load(f)
training_params.update(
{
'device': device,
'model_save_folder': os.path.join('model_save', args.method, 'labeled'),
})
safe_make_dir(training_params['model_save_folder'])
data_provider = DataProvider(batch_size=training_params['labeled']['batch_size'],
target=args.measurement)
training_params.update(
{
'input_dim': data_provider.shape_dict['gex'],
'output_dim': data_provider.shape_dict['target']
}
)
s_labeled_dataloader = data_provider.get_labeled_gex_dataloader()
labeled_dataloader_generator = data_provider.get_drug_labeled_mut_dataloader()
s_ft_evaluation_metrics = defaultdict(list)
t_ft_evaluation_metrics = defaultdict(list)
val_ft_evaluation_metrics = defaultdict(list)
test_ft_evaluation_metrics = defaultdict(list)
fold_count = 0
for train_labeled_dataloader, val_labeled_dataloader, test_labeled_dataloader in labeled_dataloader_generator:
target_regressor, train_historys = train_fn(s_dataloaders=s_labeled_dataloader,
t_dataloaders=train_labeled_dataloader,
val_dataloader=val_labeled_dataloader,
test_dataloader=test_labeled_dataloader,
metric_name=args.metric,
seed = fold_count,
**wrap_training_params(training_params, type='labeled'))
for metric in ['dpearsonr', 'dspearmanr', 'drmse', 'cpearsonr', 'cspearmanr', 'crmse']:
val_ft_evaluation_metrics[metric].append(train_historys[-2][metric][train_historys[-2]['best_index']])
test_ft_evaluation_metrics[metric].append(train_historys[-1][metric][train_historys[-2]['best_index']])
fold_count += 1
with open(os.path.join(training_params['model_save_folder'], f'test_ft_evaluation_results.json'), 'w') as f:
json.dump(test_ft_evaluation_metrics, f)
with open(os.path.join(training_params['model_save_folder'], f'ft_evaluation_results.json'), 'w') as f:
json.dump(val_ft_evaluation_metrics, f)
def init_data_providers(self):
self.dp_params['convnet'] = self
try:
self.test_data_provider = DataProvider.get_instance(
self.data_path,
self.img_size,
self.img_channels, # options i've add to cifar data provider
self.test_batch_range,
type=self.dp_type,
dp_params=self.dp_params,
test=True)
self.train_data_provider = DataProvider.get_instance(
self.data_path,
self.img_size,
self.img_channels, # options i've add to cifar data provider
self.train_batch_range,
self.model_state["epoch"],
self.model_state["batchnum"],
type=self.dp_type,
dp_params=self.dp_params,
test=False)
except DataProviderException, e:
print "Unable to create data provider: %s" % e
self.print_data_providers()
sys.exit()
def main(args):
device = 'cuda' if torch.cuda.is_available() else 'cpu'
with open(os.path.join('model_save', 'train_params.json'), 'r') as f:
training_params = json.load(f)
training_params.update(
{
'device': device,
'model_save_folder': os.path.join('model_save', 'mlp', args.omics),
})
data_provider = DataProvider(batch_size=training_params['unlabeled']['batch_size'],
target=args.measurement)
training_params.update(
{
'input_dim': data_provider.shape_dict[args.omics],
'output_dim': data_provider.shape_dict['target']
}
)
ft_evaluation_metrics = defaultdict(list)
if args.omics == 'gex':
labeled_dataloader_generator = data_provider.get_drug_labeled_gex_dataloader()
fold_count = 0
for train_labeled_dataloader, val_labeled_dataloader in labeled_dataloader_generator:
target_regressor, ft_historys = fine_tune_encoder(
train_dataloader=train_labeled_dataloader,
val_dataloader=val_labeled_dataloader,
test_dataloader=val_labeled_dataloader,
seed=fold_count,
metric_name=args.metric,
**wrap_training_params(training_params, type='labeled')
)
for metric in ['dpearsonr', 'dspearmanr','drmse', 'cpearsonr', 'cspearmanr','crmse']:
ft_evaluation_metrics[metric].append(ft_historys[-2][metric][ft_historys[-2]['best_index']])
fold_count += 1
else:
labeled_dataloader_generator = data_provider.get_drug_labeled_mut_dataloader()
fold_count = 0
test_ft_evaluation_metrics = defaultdict(list)
for train_labeled_dataloader, val_labeled_dataloader, test_labeled_dataloader in labeled_dataloader_generator:
target_regressor, ft_historys = fine_tune_encoder(
train_dataloader=train_labeled_dataloader,
val_dataloader=val_labeled_dataloader,
test_dataloader=test_labeled_dataloader,
seed=fold_count,
metric_name=args.metric,
**wrap_training_params(training_params, type='labeled')
)
for metric in ['dpearsonr', 'dspearmanr','drmse', 'cpearsonr', 'cspearmanr','crmse']:
ft_evaluation_metrics[metric].append(ft_historys[-2][metric][ft_historys[-2]['best_index']])
test_ft_evaluation_metrics[metric].append(ft_historys[-1][metric][ft_historys[-2]['best_index']])
fold_count += 1
with open(os.path.join(training_params['model_save_folder'], f'test_ft_evaluation_results.json'), 'w') as f:
json.dump(test_ft_evaluation_metrics, f)
with open(os.path.join(training_params['model_save_folder'], f'ft_evaluation_results.json'), 'w') as f:
json.dump(ft_evaluation_metrics, f)
def init_subnet_data_provider(self):
if self.output_method == 'disk':
dp = DataProvider.get_by_name('intermediate')
count = self.train_dumper.get_count()
self.train_dp = dp(self.train_output_filename, range(0, count), 'fc')
count = self.test_dumper.get_count()
self.test_dp = dp(self.test_output_filename, range(0, count), 'fc')
elif self.output_method == 'memory':
dp = DataProvider.get_by_name('memory')
self.train_dp = dp(self.train_dumper)
self.test_dp = dp(self.test_dumper)
def init_subnet_data_provider(self):
if self.output_method == 'disk':
dp = DataProvider.get_by_name('intermediate')
count = self.train_dumper.get_count()
self.train_dp = dp(self.train_output_filename, range(0, count),
'fc')
count = self.test_dumper.get_count()
self.test_dp = dp(self.test_output_filename, range(0, count), 'fc')
elif self.output_method == 'memory':
dp = DataProvider.get_by_name('memory')
self.train_dp = dp(self.train_dumper)
self.test_dp = dp(self.test_dumper)
def init_data_providers(self):
self.dp_params['convnet'] = self # dp aka dataprovider
try:
self.test_data_provider = DataProvider.get_instance(self.data_path, self.test_batch_range,
type=self.dp_type, dp_params=self.dp_params, test=True)
self.train_data_provider = DataProvider.get_instance(self.data_path, self.train_batch_range,
self.model_state["epoch"], self.model_state["batchnum"],
type=self.dp_type, dp_params=self.dp_params, test=False)
except DataProviderException, e:
print "Unable to create data provider: %s" % e
self.print_data_providers()
sys.exit()
def set_category_range(self, r):
dp = DataProvider.get_by_name(self.data_provider)
self.train_dp = dp(self.data_dir,
self.train_range,
category_range=range(r))
self.test_dp = dp(self.data_dir,
self.test_range,
category_range=range(r))
def init_data_providers(self):
self.dp_params['convnet'] = self
self.dp_params['PCA_pixel_alter'] = self.PCA_pixel_alter
self.dp_params['regress_L_channel_only'] = self.regress_L_channel_only
self.dp_params['use_local_context_ftr'] = self.use_local_context_ftr
self.dp_params['use_local_context_color_ftr'] = self.use_local_context_color_ftr
if hasattr(self,'use_position_ftr'):
self.dp_params['use_position_ftr'] = self.use_position_ftr
try:
self.test_data_provider = DataProvider.get_instance(self.libmodel, self.data_path, self.test_batch_range,
type=self.dp_type, dp_params=self.dp_params, test=DataProvider.DP_TEST)
self.train_data_provider = DataProvider.get_instance(self.libmodel, self.data_path, self.train_batch_range,
self.model_state["epoch"], self.model_state["batch_idx"],
self.model_state["epochBatchPerm"],
type=self.dp_type, dp_params=self.dp_params, test=DataProvider.DP_TRAIN)
except DataProviderException, e:
print "Unable to create data provider: %s" % e
self.print_data_providers()
sys.exit()
def init_data_providers(self):
class Dummy:
def advance_batch(self):
pass
if self.need_gpu:
ConvNet.init_data_providers(self)
if self.op.get_value("write_features_pred") or self.op.get_value("show_preds") == 2:
self.pred_data_provider = DataProvider.get_instance(self.libmodel, self.data_path, self.pred_batch_range,
type=self.dp_type, dp_params=self.dp_params, test=DataProvider.DP_PREDICT)
else:
self.train_data_provider = self.test_data_provider = Dummy()
def init_data_providers(self):
self.dp_params['convnet'] = self
self.dp_params['imgprovider'] = self.img_provider_file
try:
self.test_data_provider = DataProvider.get_instance(self.data_path_test, self.test_batch_range,
type=self.dp_type_test, dp_params=self.dp_params, test=True)
if not self.test_only:
self.train_data_provider = DataProvider.get_instance(self.data_path_train, self.train_batch_range,
self.model_state["epoch"], self.model_state["batchnum"],
type=self.dp_type_train, dp_params=self.dp_params, test=False)
self.test_batch_range = self.test_data_provider.batch_range
print "Test data provider: ", len(self.test_batch_range), " batches "
if not self.test_only:
self.train_batch_range = self.train_data_provider.batch_range
print "Training data provider: ", len(self.train_batch_range), " batches "
except DataProviderException, e:
print "Unable to create data provider: %s" % e
self.print_data_providers()
sys.exit()
def main():
hidden_units = 15
outputs = 1
inputs = 5
examples = 100
sequence_length = 10
learning_rate = 0.001
# Shapes of weights:
# Input -> hidden connections
# W_ih.shape = (inputs, hidden_units)
# b_ih.shape = (hidden_units,)
# Hidden -> hidden connections
# W_hh.shape = (hidden_units, hidden_units)
# Hidden -> output connections
# W_hk.shape = (hidden_units, outputs)
# b_hk.shape = (outputs,)
# Load trained network.
filename = r"model/trained_net.wts.npz"
W_ih, b_ih, W_hh, W_hk, b_hk = load_params(filename)
# Get training set.
d = DataProvider(examples, sequence_length, inputs, outputs)
# x.shape = (sequence_length, inputs)
# z.shape = (sequence_length, 1)
x, z = d.get_example(0)
# dictionary where key is the timestamp.
a_h = {}
b_h = dict()
b_h[-1] = np.zeros_like(b_ih)
a_k = {}
y = {}
for t in range(sequence_length):
a_h[t] = None # TODO: (3.30), don't forget bias parameter
b_h[t] = None # TODO: (3.31), hint: theta_h = tanh
a_k[t] = None # TODO: (3.32), don't forget bias parameter
y[t] = None # TODO: Binary classification
plot_predictions(y)
def init_data_providers(self):
self.dp_params['convnet'] = self
self.dp_params['imgprovider'] = self.img_provider_file
try:
if self.need_gpu:
self.test_data_provider = DataProvider.get_instance(self.data_path_test, self.test_batch_range,
type=self.dp_type_test, dp_params=self.dp_params, test=True)
self.test_batch_range = self.test_data_provider.batch_range
except Exception, e:
print "Unable to create data provider: %s" % e
self.print_data_providers()
sys.exit()
def init_data_providers(self):
class Dummy:
def advance_batch(self):
pass
if self.need_gpu:
ConvNet.init_data_providers(self)
if self.op.get_value("write_features_pred") or self.op.get_value(
"show_preds") == 2:
self.pred_data_provider = DataProvider.get_instance(
self.libmodel,
self.data_path,
self.pred_batch_range,
type=self.dp_type,
dp_params=self.dp_params,
test=DataProvider.DP_PREDICT)
else:
self.train_data_provider = self.test_data_provider = Dummy()
def test_gan(gan_type,
g_input,
g_output,
d_input,
d_output,
epochs=3,
verbose=False):
# we have to reinstantiate the generator and discriminator or the weights won't be reset
generator = Model(g_input, g_output, name='generator')
discriminator = Model(d_input, d_output, name='discriminator')
gan_model = gan_type(generator, discriminator)
gan_model.num_data = len(metadata)
gan_model.compile()
data_train = DataProvider(metadata_train)
# build a dictionary mapping between name strings and ids
data_train.class_to_id = dict((n, i) for i, n in enumerate(classes))
data_train.id_to_class = dict((i, n) for i, n in enumerate(classes))
data_test = DataProvider(metadata_test)
data_test.class_to_id = dict((n, i) for i, n in enumerate(classes))
data_test.id_to_class = dict((i, n) for i, n in enumerate(classes))
gan_model.train(epochs, data_train, data_test, verbose=verbose)
#create a checkpoint dumper
image_shape = (param_dict['image_color'], param_dict['image_size'], param_dict['image_size'], param_dict['batch_size'])
param_dict['image_shape'] = image_shape
cp_dumper = CheckpointDumper(param_dict['checkpoint_dir'], param_dict['test_id'])
param_dict['checkpoint_dumper'] = cp_dumper
#create the init_model
init_model = cp_dumper.get_checkpoint()
if init_model is None:
init_model = parse_config_file(args.param_file)
param_dict['init_model'] = init_model
#create train dataprovider and test dataprovider
dp_class = DataProvider.get_by_name(param_dict['data_provider'])
train_dp = dp_class(param_dict['data_dir'], param_dict['train_range'])
test_dp = dp_class(param_dict['data_dir'], param_dict['test_range'])
param_dict['train_dp'] = train_dp
param_dict['test_dp'] = test_dp
#get all extra information
param_dict['num_epoch'] = args.num_epoch
num_batch = util.string_to_int_list(args.num_batch)
if len(num_batch) == 1:
param_dict['num_batch'] = num_batch[0]
else:
param_dict['num_batch'] = num_batch
param_dict['num_group_list'] = util.string_to_int_list(args.num_group_list)
os.environ['THEANO_FLAGS'] = 'device=gpu'
import os
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" # see issue #152
os.environ["CUDA_VISIBLE_DEVICES"] = "1"
config = tf.ConfigProto(log_device_placement=True, allow_soft_placement=True)
config.gpu_options.allow_growth = True
session = tf.Session(config=config)
K.set_session(session)
conf = Config(flag, args[2], int(args[3]))
print(flag)
# get data
dp = DataProvider(conf)
n_terms = len(dp.idx2word)
word_embed_data = np.array(dp.word_embed)
item_embed_data = np.random.rand(dp.get_item_size(), conf.dim_word)
print("finish data processing")
# define model
word_input = Input(shape=(1, ), dtype="int32", name="word_idx")
item_pos_input = Input(shape=(1, ), dtype="int32", name="item_pos_idx")
item_neg_input = Input(shape=(1, ), dtype="int32", name="item_neg_idx")
word_embed = Embedding(output_dim=conf.dim_word,
input_dim=n_terms,
input_length=1,
name="word_embed",
for i in range(train_y_truths.shape[0])
]).item())
train_history['drmse'].append(
np.mean(
np.nanmean(np.square((train_y_truths - train_y_preds)),
axis=0)).item())
train_history['crmse'].append(
np.mean(
np.nanmean(np.square((train_y_truths - train_y_preds)),
axis=1)).item())
return (train_history, test_history)
if __name__ == '__main__':
data_provider = DataProvider()
labeled_samples, mut_only_labeled_samples = data_provider.get_labeled_samples(
)
labeled_target_df = data_provider.target_df.loc[labeled_samples]
labeled_mut_only_target_df = data_provider.target_df.loc[
mut_only_labeled_samples]
label_gex_df = data_provider.gex_dat.loc[labeled_samples]
label_mut_df = data_provider.mut_dat.loc[labeled_samples]
label_mut_only_df = data_provider.mut_dat.loc[mut_only_labeled_samples]
train_gex_data = (label_gex_df, labeled_target_df)
train_mut_data = (label_mut_df, labeled_target_df)
test_data = (label_mut_only_df, labeled_mut_only_target_df)
gex_train_history, _ = n_time_cv_regress(train_gex_data,
'gex_pred',
<span style="font-weight: bold;">Number_of_Casualties: </span>@Number_of_Casualties
</div>
<div>
<span style="font-weight: bold;">Nearest Hospital: </span>@closest_hospital_name
</div>
<div>
<span style="font-weight: bold;">Distance to @closest_hospital_name: </span>@closest_hospital_distance km
</div>
</div>
"""
COL_TPL = """
<%= get_icon(type.toLowerCase()) %> <%= type %>
"""
data_provider = DataProvider()
data_scr = data_provider.data_ds
[start_date_str, end_date_str] = data_provider.get_boundary_dates()
max_casualties = data_provider.get_max_casualties()
fa_formatter = HTMLTemplateFormatter(template=COL_TPL)
columns = [
TableColumn(field="datetime",
default_sort="descending",
title="Time",
formatter=DateFormatter(format="%Y-%m-%d %H:%M:%S")),
TableColumn(field="Verbal_severity", title="Accident_Severity", width=150),
TableColumn(field="Number_of_Casualties",
title="Number_of_Casualties",
#create a checkpoint dumper
image_shape = (param_dict['image_color'], param_dict['image_size'],
param_dict['image_size'], param_dict['batch_size'])
param_dict['image_shape'] = image_shape
cp_dumper = CheckpointDumper(param_dict['checkpoint_dir'],
param_dict['test_id'])
param_dict['checkpoint_dumper'] = cp_dumper
#create the init_model
init_model = cp_dumper.get_checkpoint()
if init_model is None:
init_model = parse_config_file(args.param_file)
param_dict['init_model'] = init_model
#create train dataprovider and test dataprovider
dp_class = DataProvider.get_by_name(param_dict['data_provider'])
train_dp = dp_class(param_dict['data_dir'], param_dict['train_range'])
test_dp = dp_class(param_dict['data_dir'], param_dict['test_range'])
param_dict['train_dp'] = train_dp
param_dict['test_dp'] = test_dp
#get all extra information
param_dict['num_epoch'] = args.num_epoch
num_batch = util.string_to_int_list(args.num_batch)
if len(num_batch) == 1:
param_dict['num_batch'] = num_batch[0]
else:
param_dict['num_batch'] = num_batch
param_dict['num_group_list'] = util.string_to_int_list(args.num_group_list)
param_dict['num_caterange_list'] = util.string_to_int_list(
def set_num_group(self, n): dp = DataProvider.get_by_name(self.data_provider) self.train_dp = dp(self.data_dir, self.train_range, n) self.test_dp = dp(self.data_dir, self.test_range, n)
def main(args, update_params_dict):
train_fn = train_ae.train_ae
device = 'cuda' if torch.cuda.is_available() else 'cpu'
with open(os.path.join('model_save', 'train_params.json'), 'r') as f:
training_params = json.load(f)
training_params['unlabeled'].update(update_params_dict)
param_str = dict_to_str(update_params_dict)
training_params.update({
'device': device,
'es_flag': False,
'retrain_flag': args.retrain_flag
})
if args.omics != 'both':
training_params.update({
'device':
device,
'model_save_folder':
os.path.join('model_save', 'ae', args.omics, param_str),
'es_flag':
False,
'retrain_flag':
args.retrain_flag
})
safe_make_dir(training_params['model_save_folder'])
task_save_folder = os.path.join('model_save', 'ae', args.omics, param_str)
safe_make_dir(task_save_folder)
random.seed(2020)
data_provider = DataProvider(
batch_size=training_params['unlabeled']['batch_size'],
target=args.measurement)
# if args.omics == 'both':
# training_params.update(
# {
# 'input_dim': sum([data_provider.shape_dict[k] for k in data_provider.shape_dict if k != 'target']),
# 'output_dim': data_provider.shape_dict['target']
# }
# )
# else:
training_params.update({'output_dim': data_provider.shape_dict['target']})
if args.omics != 'both':
training_params.update({
'input_dim':
data_provider.shape_dict[args.omics],
})
# start unlabeled training
if args.omics == 'gex':
encoder, historys = train_fn(
dataloader=data_provider.get_unlabeled_gex_dataloader(),
**wrap_training_params(training_params, type='unlabeled'))
with open(
os.path.join(training_params['model_save_folder'],
f'unlabel_train_history.pickle'), 'wb') as f:
for history in historys:
pickle.dump(dict(history), f)
elif args.omics == 'mut':
encoder, historys = train_fn(
dataloader=data_provider.get_unlabeld_mut_dataloader(match=False),
**wrap_training_params(training_params, type='unlabeled'))
with open(
os.path.join(training_params['model_save_folder'],
f'unlabel_train_history.pickle'), 'wb') as f:
for history in historys:
pickle.dump(dict(history), f)
else:
training_params.update({
'model_save_folder':
os.path.join('model_save', 'ae', 'gex', param_str),
'input_dim':
data_provider.shape_dict['gex'],
})
safe_make_dir(training_params['model_save_folder'])
gex_encoder, gex_historys = train_fn(
dataloader=data_provider.get_unlabeled_gex_dataloader(),
**wrap_training_params(training_params, type='unlabeled'))
training_params.update({
'model_save_folder':
os.path.join('model_save', 'ae', 'mut', param_str),
'input_dim':
data_provider.shape_dict['mut'],
})
safe_make_dir(training_params['model_save_folder'])
mut_encoder, mut_historys = train_fn(
dataloader=data_provider.get_unlabeld_mut_dataloader(match=False),
**wrap_training_params(training_params, type='unlabeled'))
ft_evaluation_metrics = defaultdict(list)
fold_count = 0
if args.omics == 'gex':
labeled_dataloader_generator = data_provider.get_labeled_data_generator(
omics='gex')
for train_labeled_dataloader, val_labeled_dataloader in labeled_dataloader_generator:
ft_encoder = deepcopy(encoder)
target_regressor, ft_historys = fine_tuning.fine_tune_encoder(
encoder=ft_encoder,
train_dataloader=train_labeled_dataloader,
val_dataloader=val_labeled_dataloader,
test_dataloader=val_labeled_dataloader,
seed=fold_count,
metric_name=args.metric,
task_save_folder=task_save_folder,
**wrap_training_params(training_params, type='labeled'))
for metric in [
'dpearsonr', 'dspearmanr', 'drmse', 'cpearsonr',
'cspearmanr', 'crmse'
]:
ft_evaluation_metrics[metric].append(
ft_historys[-2][metric][ft_historys[-2]['best_index']])
fold_count += 1
elif args.omics == 'mut':
labeled_dataloader_generator = data_provider.get_labeled_data_generator(
omics='mut')
test_ft_evaluation_metrics = defaultdict(list)
for train_labeled_dataloader, val_labeled_dataloader, test_labeled_dataloader in labeled_dataloader_generator:
ft_encoder = deepcopy(encoder)
target_regressor, ft_historys = fine_tuning.fine_tune_encoder(
encoder=ft_encoder,
train_dataloader=train_labeled_dataloader,
val_dataloader=val_labeled_dataloader,
test_dataloader=test_labeled_dataloader,
seed=fold_count,
metric_name=args.metric,
task_save_folder=task_save_folder,
**wrap_training_params(training_params, type='labeled'))
for metric in [
'dpearsonr', 'dspearmanr', 'drmse', 'cpearsonr',
'cspearmanr', 'crmse'
]:
ft_evaluation_metrics[metric].append(
ft_historys[-2][metric][ft_historys[-2]['best_index']])
test_ft_evaluation_metrics[metric].append(
ft_historys[-1][metric][ft_historys[-2]['best_index']])
fold_count += 1
with open(
os.path.join(task_save_folder,
f'{param_str}_test_ft_evaluation_results.json'),
'w') as f:
json.dump(test_ft_evaluation_metrics, f)
else:
labeled_dataloader_generator = data_provider.get_labeled_data_generator(
omics='both')
for train_labeled_dataloader, val_labeled_dataloader in labeled_dataloader_generator:
ft_gex_encoder = deepcopy(gex_encoder)
ft_mut_encoder = deepcopy(mut_encoder)
target_regressor, ft_historys = fine_tuning_mul.fine_tune_encoder(
encoders=[ft_gex_encoder, ft_mut_encoder],
train_dataloader=train_labeled_dataloader,
val_dataloader=val_labeled_dataloader,
test_dataloader=val_labeled_dataloader,
seed=fold_count,
metric_name=args.metric,
task_save_folder=task_save_folder,
**wrap_training_params(training_params, type='labeled'))
for metric in [
'dpearsonr', 'dspearmanr', 'drmse', 'cpearsonr',
'cspearmanr', 'crmse'
]:
ft_evaluation_metrics[metric].append(
ft_historys[-2][metric][ft_historys[-2]['best_index']])
fold_count += 1
with open(
os.path.join(task_save_folder,
f'{param_str}_ft_evaluation_results.json'), 'w') as f:
json.dump(ft_evaluation_metrics, f)
def main():
global cfg, global_all_step
global_all_step = 0
cfg = DefaultConfig()
args = {
'resnet18': RESNET18_SUNRGBD_CONFIG().args(),
}
# Setting random seed
# if cfg.MANUAL_SEED is None:
# cfg.MANUAL_SEED = random.randint(1, 10000)
# random.seed(cfg.MANUAL_SEED)
# torch.manual_seed(cfg.MANUAL_SEED)
# args for different backbones
cfg.parse(args['resnet18'])
run_id = random.randint(1, 100000)
summary_dir = '/home/lzy/summary/generateDepth/' + 'train_depth_1e-3_0909_' + str(
run_id)
if not os.path.exists(summary_dir):
os.mkdir(summary_dir)
writer = SummaryWriter(summary_dir)
cfg.LR = 0.001
os.environ["CUDA_VISIBLE_DEVICES"] = '0,1,2'
device_ids = torch.cuda.device_count()
print('device_ids:', device_ids)
# project_name = reduce(lambda x, y: str(x) + '/' + str(y), os.path.realpath(__file__).split(os.sep)[:-1])
# util.mkdir('logs')
# normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
# std=[0.229, 0.224, 0.225])
# normalize=transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])
# data
train_dataset = dataset.SingleDataset(
cfg,
data_dir=cfg.DATA_DIR_TRAIN,
transform=transforms.Compose([
dataset.Resize((cfg.LOAD_SIZE, cfg.LOAD_SIZE)),
dataset.RandomCrop((cfg.FINE_SIZE, cfg.FINE_SIZE)),
dataset.RandomHorizontalFlip(),
# dataset.RandomRotate(),
# dataset.RandomFlip(),
# dataset.PILBrightness(0.4),
# dataset.PILContrast(0.4),
# dataset.PILColorBalance(0.4),
# dataset.PILSharpness(0.4),
dataset.ToTensor(),
dataset.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
]))
val_dataset = dataset.SingleDataset(
cfg,
data_dir=cfg.DATA_DIR_VAL,
transform=transforms.Compose([
dataset.Resize((cfg.LOAD_SIZE, cfg.LOAD_SIZE)),
dataset.CenterCrop((cfg.FINE_SIZE, cfg.FINE_SIZE)),
dataset.ToTensor(),
dataset.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
]))
# train_loader = torch.utils.data.DataLoader(
# train_dataset, batch_size=cfg.BATCH_SIZE, shuffle=True,
# num_workers=4, pin_memory=True, sampler=None)
# val_loader = torch.utils.data.DataLoader(
# val_dataset,batch_size=16, shuffle=False,
# num_workers=4, pin_memory=True)
train_loader = DataProvider(cfg,
dataset=train_dataset,
batch_size=150,
shuffle=True)
val_loader = DataProvider(cfg,
dataset=val_dataset,
batch_size=120,
shuffle=False)
# class weights
# num_classes_train = list(Counter([i[1] for i in train_loader.dataset.imgs]).values())
# cfg.CLASS_WEIGHTS_TRAIN = torch.FloatTensor(num_classes_train)
# writer = SummaryWriter(log_dir=cfg.LOG_PATH) # tensorboard
# net_classification_1=resnet50(pretrained=True)
# net_classification_2=resnet50(pretrained=True)
net_classification_2 = models.__dict__['resnet50'](num_classes=365)
# net_classification_2=torchvision.models.resnext101_32x8d(pretrained=True, progress=True)
# net_classification_2 = models.__dict__['resnet18'](num_classes=365)
# net_classification_2=torch.hub.load('facebookresearch/WSL-Images', 'resnext101_32x16d_wsl')
net_classification_2.fc = nn.Linear(2048, 19)
# net_classification_2=torch.hub.load('facebookresearch/WSL-Images', 'resnext101_32x16d_wsl')
# for param in net_classification_1.parameters():
# param.requires_grad = False
# for param in net_classification_2.parameters():
# param.requires_grad = True
# net_classification_1.fc = nn.Sequential(nn.Dropout(p=0.5),nn.Linear(2048, 1024),nn.LeakyReLU(inplace=True),nn.Linear(1024,67))
# net_classification_2.fc = nn.Sequential(nn.Dropout(p=0.5),nn.Linear(2048,1024),nn.LeakyReLU(inplace=True),nn.Linear(1024,67))
# net_classification_1.load_state_dict(torch.load("./bestmodel/best_model_resnext_16d_2048_1024_dropout_0.5_b.pkl"))、
# net_classification_2.load_state_dict(torch.load("./bestmodel/best_model_resnext_16d_2048_1024_dropout_0.5_b.pkl"))
# net_classification_2
load_path = "resnet50_Depth_sunrgbd_best_0909_5e-4_.pth.tar"
checkpoint = torch.load(load_path,
map_location=lambda storage, loc: storage)
state_dict = {
str.replace(k, 'module.', ''): v
for k, v in checkpoint['state_dict'].items()
}
best_mean_depth = checkpoint['best_mean_2']
print("load sunrgbd dataset:", best_mean_depth)
net_classification_2.load_state_dict(state_dict)
# print(net_classification_1)
# num_ftrs = net_classification_1.fc.in_features
# net_classification_1.fc = nn.Linear(num_ftrs, cfg.NUM_CLASSES)
# num_ftrs = net_classification_2.fc.in_features
# net_classification_2.fc = nn.Linear(num_ftrs, cfg.NUM_CLASSES)
# net_classification_2.fc = nn.Sequential(nn.Dropout(p=0.5),nn.Linear(2048,1024),nn.LeakyReLU(inplace=True),nn.Linear(1024,67))
net_classification_2.fc = nn.Linear(2048, 67)
init_weights(net_classification_2.fc, 'normal')
print(net_classification_2)
# net_classification_1.cuda()
net_classification_2.cuda()
cudnn.benchmark = True
if cfg.GENERATE_Depth_DATA:
print('GENERATE_Depth_DATA model set')
cfg_generate = copy.deepcopy(cfg)
cfg_generate.CHECKPOINTS_DIR = '/home/lzy/generateDepth/bestmodel/before_best.pth'
cfg_generate.GENERATE_Depth_DATA = False
cfg_generate.NO_UPSAMPLE = False
checkpoint = torch.load(cfg_generate.CHECKPOINTS_DIR)
model = define_TrecgNet(cfg_generate, upsample=True, generate=True)
load_checkpoint_depth(model,
cfg_generate.CHECKPOINTS_DIR,
checkpoint,
data_para=True)
generate_model = torch.nn.DataParallel(model).cuda()
generate_model.eval()
# net_classification_1 = torch.nn.DataParallel(net_classification_1).cuda()
net_classification_2 = torch.nn.DataParallel(net_classification_2).cuda()
criterion = nn.CrossEntropyLoss().cuda()
# best_mean_1=0
best_mean_2 = 0
# optimizer_2 = torch.optim.Adam(net_classification_2.parameters(), lr=cfg.LR, betas=(0.5, 0.999))
optimizer_2 = torch.optim.SGD(net_classification_2.parameters(),
lr=cfg.LR,
momentum=cfg.MOMENTUM,
weight_decay=cfg.WEIGHT_DECAY)
schedulers = get_scheduler(optimizer_2, cfg, cfg.LR_POLICY)
for epoch in range(0, 1000):
if global_all_step > cfg.NITER_TOTAL:
break
# meanacc_1,meanacc_2=validate(val_loader, net_classification_1,net_classification_2,generate_model,criterion,epoch)
# for param_group in optimizer_2.param_groups:
# lr_t = param_group['lr']
# print('/////////learning rate = %.7f' % lr_t)
# writer.add_scalar('LR',lr_t,global_step=epoch)
printlr(optimizer_2, writer, epoch)
train(train_loader, schedulers, net_classification_2, generate_model,
criterion, optimizer_2, epoch, writer)
meanacc_2 = validate(val_loader, net_classification_2, generate_model,
criterion, epoch, writer)
# meanacc_2=validate(val_loader,net_classification_2,generate_model,criterion,epoch,writer)
# train(train_loader,net_classification_2,generate_model,criterion,optimizer_2,epoch,writer)
# meanacc_2=validate(val_loader,net_classification_2,generate_model,criterion,epoch,writer)
# writer.add_image(depth_image[0])
# save best
# if meanacc_1>best_mean_1:
# best_mean_1=meanacc_1
# print('best_mean_color:',str(best_mean_1))
# save_checkpoint({
# 'epoch': epoch,
# 'arch': cfg.ARCH,
# 'state_dict': net_classification_1.state_dict(),
# 'best_mean_1': best_mean_1,
# 'optimizer' : optimizer_1.state_dict(),
# },CorD=True)
if meanacc_2 > best_mean_2:
best_mean_2 = meanacc_2
print('best_mean_depth:', str(best_mean_2))
save_checkpoint(
{
'epoch': epoch,
'arch': cfg.ARCH,
'state_dict': net_classification_2.state_dict(),
'best_mean_2': best_mean_2,
'optimizer': optimizer_2.state_dict(),
},
CorD=False)
# print('best_mean_color:',str(best_mean_1))
# writer.add_scalar('mean_acc_color', meanacc_1, global_step=epoch)
writer.add_scalar('mean_acc_depth', meanacc_2, global_step=epoch)
# writer.add_scalar('best_meanacc_color', best_mean_1, global_step=epoch)
writer.add_scalar('best_meanacc_depth', best_mean_2, global_step=epoch)
writer.close()
def init_data_provider(self): self.train_dp = DataProvider(self.batch_size, self.data_dir, self.train_range) self.test_dp = DataProvider(self.batch_size, self.data_dir, self.test_range)
class Trainer:
CHECKPOINT_REGEX = None
def __init__(self, test_id, data_dir, checkpoint_dir, train_range, test_range, test_freq,
save_freq, batch_size, num_epoch, image_size, image_color, learning_rate, n_out,
autoInit=True, adjust_freq = 1, factor = 1.0):
self.test_id = test_id
self.data_dir = data_dir
self.checkpoint_dir = checkpoint_dir
self.train_range = train_range
self.test_range = test_range
self.test_freq = test_freq
self.save_freq = save_freq
self.batch_size = batch_size
self.num_epoch = num_epoch
self.image_size = image_size
self.image_color = image_color
self.learning_rate = learning_rate
self.n_out = n_out
self.factor = factor
self.adjust_freq = adjust_freq
self.regex = re.compile('^test%d-(\d+)\.(\d+)$' % self.test_id)
self.init_data_provider()
self.image_shape = (self.batch_size, self.image_color, self.image_size, self.image_size)
self.train_outputs = []
self.test_outputs = []
self.net = FastNet(self.learning_rate, self.image_shape, self.n_out, autoAdd = autoInit)
self.num_batch = self.curr_epoch = self.curr_batch = 0
self.train_data = None
self.test_data = None
self.num_train_minibatch = 0
self.num_test_minibatch = 0
self.checkpoint_file = ''
def init_data_provider(self):
self.train_dp = DataProvider(self.batch_size, self.data_dir, self.train_range)
self.test_dp = DataProvider(self.batch_size, self.data_dir, self.test_range)
def get_next_minibatch(self, i, train = TRAIN):
if train == TRAIN:
num = self.num_train_minibatch
data = self.train_data
else:
num = self.num_test_minibatch
data = self.test_data
batch_data = data['data']
batch_label = data['labels']
batch_size = self.batch_size
if i == num -1:
input = batch_data[:, i * batch_size: -1]
label = batch_label[i* batch_size : -1]
else:
input = batch_data[:, i * batch_size: (i +1)* batch_size]
label = batch_label[i * batch_size: (i + 1) * batch_size]
return input, label
def save_checkpoint(self):
model = {}
model['batchnum'] = self.train_dp.get_batch_num()
model['epoch'] = self.num_epoch + 1
model['layers'] = self.net.get_dumped_layers()
model['train_outputs'] = self.train_outputs
model['test_outputs'] = self.test_outputs
dic = {'model_state': model, 'op':None}
saved_filename = [f for f in os.listdir(self.checkpoint_dir) if self.regex.match(f)]
for f in saved_filename:
os.remove(os.path.join(self.checkpoint_dir, f))
checkpoint_filename = "test%d-%d.%d" % (self.test_id, self.curr_epoch, self.curr_batch)
checkpoint_file_path = os.path.join(self.checkpoint_dir, checkpoint_filename)
self.checkpoint_file = checkpoint_file_path
print checkpoint_file_path
with open(checkpoint_file_path, 'w') as f:
cPickle.dump(dic, f)
def get_test_error(self):
start = time.time()
_, _, self.test_data = self.test_dp.get_next_batch()
self.num_test_minibatch = ceil(self.test_data['data'].shape[1], self.batch_size)
for i in range(self.num_test_minibatch):
input, label = self.get_next_minibatch(i, TEST)
label = np.array(label).astype(np.float32)
label.shape = (label.size, 1)
self.net.train_batch(input, label, TEST)
cost , correct, numCase, = self.net.get_batch_information()
self.test_outputs += [({'logprob': [cost, 1-correct]}, numCase, time.time() - start)]
print 'error: %f logreg: %f time: %f' % (1-correct, cost, time.time() -
start)
def check_continue_trainning(self):
return self.curr_epoch <= self.num_epoch
def check_test_data(self):
return self.num_batch % self.test_freq == 0
def check_save_checkpoint(self):
return self.num_batch % self.save_freq == 0
def check_adjust_lr(self):
return self.num_batch % self.adjust_freq == 0
def train(self):
self.curr_epoch, self.curr_batch, self.train_data = self.train_dp.get_next_batch()#self.train_dp.wait()
while self.check_continue_trainning():
start = time.time()
self.num_train_minibatch = ceil(self.train_data['data'].shape[1], self.batch_size)
for i in range(self.num_train_minibatch):
input, label = self.get_next_minibatch(i)
self.net.train_batch(input, label)
cost , correct, numCase = self.net.get_batch_information()
self.train_outputs += [({'logprob': [cost, 1-correct]}, numCase, time.time() - start)]
print '%d.%d: error: %f logreg: %f time: %f' % (self.curr_epoch, self.curr_batch, 1-correct, cost, time.time() -
start)
self.num_batch += 1
if self.check_test_data():
print '---- test ----'
self.get_test_error()
print '------------'
if self.factor != 1.0 and self.check_adjust_lr():
print '---- adjust learning rate ----'
self.net.adjust_learning_rate(self.factor)
print '--------'
if self.check_save_checkpoint():
print '---- save checkpoint ----'
self.save_checkpoint()
print '------------'
self.curr_epoch, self.curr_batch, self.train_data = self.train_dp.get_next_batch()##self.train_dp.wait()
if self.num_batch % self.save_freq != 0:
print '---- save checkpoint ----'
self.save_checkpoint()
def build_encoder(args):
train_fn = train_vae.train_vae
device = 'cuda' if torch.cuda.is_available() else 'cpu'
with open(os.path.join('model_save', 'train_params.json'), 'r') as f:
training_params = json.load(f)
parsed_ft_params = parsing_utils.parse_hyper_vae_ft_evaluation_result(
metric_name=args.metric)
training_params['unlabeled'].update(parsed_ft_params[0])
training_params['labeled']['train_num_epochs'] = parsed_ft_params[1]
training_params.update({
'device':
device,
'model_save_folder':
os.path.join('model_save', 'ae500'),
'es_flag':
False,
'retrain_flag':
True
})
task_save_folder = training_params['model_save_folder']
safe_make_dir(training_params['model_save_folder'])
random.seed(2020)
data_provider = DataProvider(
batch_size=training_params['unlabeled']['batch_size'],
target=args.measurement)
training_params.update({
'input_dim': data_provider.shape_dict['gex'],
'output_dim': data_provider.shape_dict['target']
})
encoder, historys = train_fn(
dataloader=data_provider.get_unlabeled_gex_dataloader(),
**wrap_training_params(training_params, type='unlabeled'))
with open(
os.path.join(training_params['model_save_folder'],
f'unlabel_train_history.pickle'), 'wb') as f:
for history in historys:
pickle.dump(dict(history), f)
ft_evaluation_metrics = defaultdict(list)
labeled_dataloader = data_provider.get_labeled_gex_dataloader()
ft_encoder = deepcopy(encoder)
target_regressor, ft_historys = fine_tuning.fine_tune_encoder_new(
encoder=ft_encoder,
train_dataloader=labeled_dataloader,
val_dataloader=labeled_dataloader,
test_dataloader=None,
seed=2021,
metric_name=args.metric,
task_save_folder=task_save_folder,
**wrap_training_params(training_params, type='labeled'))
for metric in ['dpearsonr', 'drmse', 'cpearsonr', 'crmse']:
try:
ft_evaluation_metrics[metric].append(ft_historys[-2][metric][-1])
except:
pass
with open(os.path.join(task_save_folder, f'ft_evaluation_results.json'),
'w') as f:
json.dump(ft_evaluation_metrics, f)
torch.save(target_regressor.encoder.state_dict(),
os.path.join('model_save', 'reference_encoder.pt'))
cfg,
data_dir=cfg.DATA_DIR_TRAIN,
transform=transforms.Compose(train_transforms))
val_dataset = SPL10.SPL10_Dataset(
cfg,
data_dir=cfg.DATA_DIR_VAL,
transform=transforms.Compose([
SPL10.Resize((cfg.LOAD_SIZE, cfg.LOAD_SIZE)),
SPL10.CenterCrop((cfg.FINE_SIZE, cfg.FINE_SIZE)),
RGB2Lab(),
SPL10.ToTensor(),
SPL10.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
]))
train_loader = DataProvider(cfg, dataset=train_dataset)
val_loader = DataProvider(cfg, dataset=val_dataset, shuffle=False)
num_classes_train = list(
Counter([i[1] for i in train_loader.dataset.imgs]).values())
cfg.CLASS_WEIGHTS_TRAIN = torch.FloatTensor(num_classes_train)
model = Contrastive_CrossModal_Conc(cfg, device=device)
model = nn.DataParallel(model).to(device)
optim = Adam(model.parameters(), lr=cfg.LR)
load_model = False
if load_model:
model = torch.load('./checkpoint/model_1_LAB.mdl')
print("load pretrained model")
# loss_optim = Adam(infomax_fn.parameters(), lr=2e-4)
# cls_criterion = torch.nn.CrossEntropyLoss(cfg.CLASS_WEIGHTS_TRAIN.to(device))
def init_subnet_data_provider(self):
dp = DataProvider.get_by_name('intermediate')
count = self.train_dumper.get_count()
self.train_dp = dp(self.train_output_filename, range(0, count), 'fc')
count = self.test_dumper.get_count()
self.test_dp = dp(self.test_output_filename, range(0, count), 'fc')
def set_category_range(self, r): dp = DataProvider.get_by_name(self.data_provider) self.train_dp = dp(self.data_dir, self.train_range, category_range = range(r)) self.test_dp = dp(self.data_dir, self.test_range, category_range = range(r))
from config import Config
from data import DataProvider
from gensim.models.word2vec import Word2Vec
import numpy as np
import os
flag = "tag"
conf = Config(flag, "tag" , 300)
if not os.path.exists(conf.path_word_w2c) and not os.path.exists(conf.path_doc_w2c):
doc_embed = np.load(conf.path_doc_npy + ".npy")[0]
dp = DataProvider(conf)
# generate doc embedding file
f = open(conf.path_doc_w2c,"w")
f.write(str(len(dp.idx2prod)))
f.write(" ")
f.write(str(conf.dim_item))
f.write("\n")
idx = 0
batch = ""
for word in dp.idx2prod:
batch = "".join([batch, word])
batch = "".join([batch, " "])
for i in range(conf.dim_item):
batch = "".join([batch, str(doc_embed[idx][i])])
batch = "".join([batch, " "])
batch = "".join([batch, "\n"])
idx += 1
def main():
global cfg
cfg = DefaultConfig()
args = {
'resnet18': RESNET18_SUNRGBD_CONFIG().args(),
}
# args for different backbones
cfg.parse(args['resnet18'])
cfg.LR = 1e-4
cfg.EPOCHS = 200
# print('cfg.EPOCHS:',cfg.EPOCHS)
os.environ["CUDA_VISIBLE_DEVICES"] = '0,1'
# normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
# std=[0.229, 0.224, 0.225])
# dataset.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])
# data
train_dataset = dataset.SingleDataset(
cfg,
data_dir=cfg.DATA_DIR_TRAIN,
transform=transforms.Compose([
dataset.Resize((cfg.LOAD_SIZE, cfg.LOAD_SIZE)),
dataset.RandomCrop((cfg.FINE_SIZE, cfg.FINE_SIZE)),
dataset.RandomHorizontalFlip(),
dataset.ToTensor(),
dataset.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
]))
val_dataset = dataset.SingleDataset(
cfg,
data_dir=cfg.DATA_DIR_VAL,
transform=transforms.Compose([
dataset.Resize((cfg.LOAD_SIZE, cfg.LOAD_SIZE)),
dataset.CenterCrop((cfg.FINE_SIZE, cfg.FINE_SIZE)),
dataset.ToTensor(),
dataset.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
]))
# train_loader = torch.utils.data.DataLoader(
# train_dataset, batch_size=cfg.BATCH_SIZE, shuffle=True,
# num_workers=4, pin_memory=True, sampler=None)
# val_loader = torch.utils.data.DataLoader(
# val_dataset,batch_size=cfg.BATCH_SIZE, shuffle=False,
# num_workers=4, pin_memory=True)
train_loader = DataProvider(cfg,
dataset=train_dataset,
batch_size=20,
shuffle=True)
val_loader = DataProvider(cfg,
dataset=val_dataset,
batch_size=5,
shuffle=False)
run_id = random.randint(1, 100000)
summary_dir = '/home/lzy/summary/generateDepth/' + 'finetuning_nofix_' + str(
run_id)
if not os.path.exists(summary_dir):
os.mkdir(summary_dir)
writer = SummaryWriter(summary_dir)
if cfg.GENERATE_Depth_DATA:
print('GENERATE_Depth_DATA model set')
cfg_generate = copy.deepcopy(cfg)
cfg_generate.CHECKPOINTS_DIR = '/home/lzy/generateDepth/checkpoints/best_AtoB/trecg_AtoB_best.pth'
cfg_generate.GENERATE_Depth_DATA = False
cfg_generate.NO_UPSAMPLE = False
checkpoint = torch.load(cfg_generate.CHECKPOINTS_DIR)
model = define_TrecgNet(cfg_generate, upsample=True, generate=True)
load_checkpoint_depth(model,
cfg_generate.CHECKPOINTS_DIR,
checkpoint,
data_para=True)
generate_model = torch.nn.DataParallel(model).cuda()
generate_model.eval()
model = ReD_Model(cfg)
policies = model.get_optim_policies()
model = torch.nn.DataParallel(model).cuda()
criterion = nn.CrossEntropyLoss().cuda()
best_mean = 0
# optimizer= torch.optim.Adam(policies, cfg.LR,
# weight_decay=cfg.WEIGHT_DECAY)
optimizer = torch.optim.SGD(policies,
cfg.LR,
momentum=cfg.MOMENTUM,
weight_decay=cfg.WEIGHT_DECAY)
for epoch in range(cfg.START_EPOCH, cfg.EPOCHS + 1):
adjust_learning_rate(optimizer, epoch)
# mean_acc=validate(val_loader,model,criterion,generate_model,epoch,writer)
train(train_loader, model, criterion, generate_model, optimizer, epoch,
writer)
mean_acc = validate(val_loader, model, criterion, generate_model,
epoch, writer)
if mean_acc > best_mean:
best_mean = mean_acc
print('best mean accuracy:', best_mean)
else:
print('best mean accuracy:', best_mean)
writer.add_scalar('mean_acc_color', mean_acc, global_step=epoch)
writer.add_scalar('best_meanacc_color', best_mean, global_step=epoch)
writer.close()
def init_data_provider(self): dp = DataProvider.get_by_name(self.data_provider) self.train_dp = dp(self.data_dir, self.train_range) self.test_dp = dp(self.data_dir, self.test_range)
def init_subnet_data_provider(self):
dp = DataProvider.get_by_name('intermediate')
count = self.train_dumper.get_count()
self.train_dp = dp(self.train_output_filename, range(0, count), 'fc')
count = self.test_dumper.get_count()
self.test_dp = dp(self.test_output_filename, range(0, count), 'fc')
if os.path.exists(_CUR_LOG_FILE_NAME):
os.rename(_CUR_LOG_FILE_NAME, _PRED_LOG_FILE_NAME)
log.basicConfig( level=log.INFO)
# log.basicConfig(filename=_CUR_LOG_FILE_NAME, level=log.INFO)
log.info('start')
if __name__ == '__main__':
# try:
FILE_NAME = "data.dat"
INIT_DATA_FILE_NAME = 'init.dat'
TEST_FILE_NAME = "test_data.dat"
TEST_INIT_DATA_FILE_NAME = 'test_init.dat'
# binder.bind(str, annotated_with="data_file_name", to_instance = FILE_NAME)
# binder.bind(str, annotated_with="init_file_name", to_instance = INIT_DATA_FILE_NAME)
_init_log()
dataProvider = DataProvider(FILE_NAME, INIT_DATA_FILE_NAME)
# test_data = DataProvider(TEST_FILE_NAME, TEST_INIT_DATA_FILE_NAME).get_data()
# assert(calc_class_re())
app = QApplication(sys.argv)
data = dataProvider.get_data()
# print data
classifier = c45(data, max_repeat_var=10)
form = MainWindow(data, classifier)
## print data
# classifier = c45(data, max_repeat_var=10)
# pos_sum = 0
# for row, target in zip(data.data, data.target):
# pos = 0
# for l, c in classifier.get_labels_count(row).items():
# pos += 1
def set_num_group(self, n):
dp = DataProvider.get_by_name(self.data_provider)
self.train_dp = dp(self.data_dir, self.train_range, n)
self.test_dp = dp(self.data_dir, self.test_range, n)
os.path.realpath(__file__).split(os.sep)[:-1])
util.mkdir('logs')
val_dataset = dataset.AlignedConcDataset(
cfg,
data_dir=cfg.DATA_DIR_VAL,
transform=transforms.Compose([
dataset.Resize((cfg.LOAD_SIZE, cfg.LOAD_SIZE)),
dataset.CenterCrop((cfg.FINE_SIZE, cfg.FINE_SIZE)),
dataset.ToTensor(),
dataset.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])
]))
batch_size_val = cfg.BATCH_SIZE
val_loader = DataProvider(cfg,
dataset=val_dataset,
batch_size=batch_size_val,
shuffle=False)
writer = SummaryWriter(log_dir=cfg.LOG_PATH) # tensorboard
model = TRecgNet_Upsample_Resiual(cfg, writer)
model.set_data_loader(None, val_loader, None)
model.net = nn.DataParallel(model.net).to(model.device)
model.set_log_data(cfg)
def evaluate():
checkpoint_path = os.path.join(cfg.CHECKPOINTS_DIR, cfg.RESUME_PATH)
checkpoint = torch.load(checkpoint_path)
load_epoch = checkpoint['epoch']
model.load_checkpoint(model.net,
checkpoint_path,
logger.info('read text')
train_text, test_text = get_text()
args = parse_args()
seq_length = args.seq_length
batch_size = args.batch_size
rnn_size = args.rnn_size
context_size = args.context_size
num_epochs = args.num_epoch
learning_rate = args.lr
alpha = args.alpha
logger.info('build vocabulary')
data_provider = DataProvider(train_text, seq_length, batch_size, logger)
X_train, y_train = data_provider.get_data()
logger.info('X.shape={}, y.shape={}'.format(X_train.shape, y_train.shape))
vocab_size = data_provider.vocab_size
input_data = tf.placeholder(tf.float32,
[batch_size, seq_length, vocab_size])
targets = tf.placeholder(tf.float32, [batch_size, vocab_size])
test_data_provider = DataProvider(test_text, seq_length, batch_size,
logger, data_provider.vocab)
if args.cell == 'lstm':
cell = LSTMCell(num_units=rnn_size)
# Model
net = Net()
net.push(Conv2d(5, 5, 1, 6)) # 1x28x28 -> 6x24x24
net.push(Relu())
net.push(Maxpooling(2, 2)) # 6x24x24 -> 6x12x12
net.push(Conv2d(5, 5, 6, 16)) # 6x12x12 -> 16x8x8
net.push(Relu())
net.push(Maxpooling(2, 2)) # 16x8x8 -> 16x4x4
net.push(Reshape((256)))
net.push(Linear(256, 84))
net.push(Relu())
net.push(Softmax(84, 10))
# Data
data = DataProvider()
n = 10000
data.train_input(x[:n], y[:n])
data.test_input(xt, yt)
data.batch_size(16)
lr = 0.0009
gamma = 0.9
for epoch in xrange(50):
print 'Epoch: ', epoch
# Training (Mini-batch)
now = time.time()
for _ in xrange(data.batch_run()):
net.input(data.next_batch())
net.forward()
from config import Config
from data import DataProvider
from gensim.models.word2vec import Word2Vec
import numpy as np
import os
flag = "tag"
conf = Config(flag, "tag", 300)
if not os.path.exists(conf.path_word_w2c) and not os.path.exists(
conf.path_doc_w2c):
doc_embed = np.load(conf.path_doc_npy + ".npy")[0]
dp = DataProvider(conf)
# generate doc embedding file
f = open(conf.path_doc_w2c, "w")
f.write(str(len(dp.idx2prod)))
f.write(" ")
f.write(str(conf.dim_item))
f.write("\n")
idx = 0
batch = ""
for word in dp.idx2prod:
batch = "".join([batch, word])
batch = "".join([batch, " "])
for i in range(conf.dim_item):
batch = "".join([batch, str(doc_embed[idx][i])])
batch = "".join([batch, " "])
batch = "".join([batch, "\n"])
os.environ['THEANO_FLAGS'] = 'device=gpu'
import os
os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID" # see issue #152
os.environ["CUDA_VISIBLE_DEVICES"]="1"
config = tf.ConfigProto(log_device_placement=True, allow_soft_placement=True)
config.gpu_options.allow_growth = True
session = tf.Session(config=config)
K.set_session(session)
conf = Config(flag, args[2], int(args[3]))
print(flag)
# get data
dp = DataProvider(conf)
n_terms = len(dp.idx2word)
word_embed_data = np.array(dp.word_embed)
item_embed_data = np.random.rand(dp.get_item_size(), conf.dim_word)
print("finish data processing")
# define model
word_input = Input(shape=(1,), dtype ="int32", name ="word_idx")
item_pos_input = Input(shape=(1,), dtype ="int32", name ="item_pos_idx")
item_neg_input = Input(shape=(1,), dtype ="int32", name ="item_neg_idx")
word_embed = Embedding(output_dim=conf.dim_word, input_dim=n_terms, input_length=1, name="word_embed",
weights=[word_embed_data], trainable=False)
item_embed = Embedding(output_dim=conf.dim_word, input_dim=dp.get_item_size(), input_length=1, name="item_embed",
weights=[item_embed_data], trainable=True)
class Net():
def __init__(self,
net,
snapshot_prefix,
dp_params,
preproc,
iter=None,
test=False):
if iter is not None:
self.N = caffe.Net(net,
snapshot_prefix + str(iter) + '.caffemodel',
caffe.TRAIN)
self.iter = iter
else:
if test:
self.N = caffe.Net(net, snapshot_prefix, caffe.TEST)
# self.N = caffe.Net(net, caffe.TRAIN)
else:
self.N = caffe.Net(net, snapshot_prefix, caffe.TRAIN)
self.iter = 0
# Data provider
self.dp = DataProvider(dp_params, preproc)
self.bsize = self.dp.batch_size
self.prevs = {}
self.test = test
def forward(self):
ind = self.iter * self.bsize
if self.test:
_data, _labels = self.dp.get_batch_test(ind)
else:
_data, _labels = self.dp.get_batch(ind)
# set data as input
self.N.blobs['data'].data[...] = _data
for label_key in _labels.keys():
self.N.blobs[label_key].data[...] = _labels[label_key].reshape(
self.N.blobs[label_key].data.shape)
# Forward
# t0 = time.time()
out = self.N.forward()
self.iter += 1
return out
def backward(self):
self.N.backward()
# update filter parameters
# t0 = time.time()
for layer_name, lay in zip(self.N._layer_names, self.N.layers):
for blobind, blob in enumerate(lay.blobs):
diff = blob.diff[:]
key = (layer_name, blobind)
if key in self.prevs:
previous_change = self.prevs[key]
else:
previous_change = 0
lr = 0.01
wd = 0.0005
momentum = 0.9
if blobind == 1:
lr = 2 * lr
wd = 0
if lay.type == "BatchNorm":
lr = 0
wd = 0
change = momentum * previous_change - lr * diff - lr * wd * blob.data[:]
blob.data[:] += change
self.prevs[key] = change
def empty_diff(self):
for layer_name, lay in zip(self.N._layer_names, self.N.layers):
for blobind, blob in enumerate(lay.blobs):
blob.diff[:] = 0
def main():
global cfg
cfg = DefaultConfig()
args = {
'resnet18': RESNET18_SUNRGBD_CONFIG().args(),
}
# Setting random seed
# if cfg.MANUAL_SEED is None:
# cfg.MANUAL_SEED = random.randint(1, 10000)
# random.seed(cfg.MANUAL_SEED)
# torch.manual_seed(cfg.MANUAL_SEED)
# args for different backbones
cfg.parse(args['resnet18'])
run_id = random.randint(1, 100000)
summary_dir = '/home/lzy/summary/generateDepth/' + 'train_rgb_' + str(
run_id)
if not os.path.exists(summary_dir):
os.mkdir(summary_dir)
writer = SummaryWriter(summary_dir)
cfg.LR = 0.0001
os.environ["CUDA_VISIBLE_DEVICES"] = '1,2'
device_ids = torch.cuda.device_count()
print('device_ids:', device_ids)
# project_name = reduce(lambda x, y: str(x) + '/' + str(y), os.path.realpath(__file__).split(os.sep)[:-1])
# util.mkdir('logs')
# normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
# std=[0.229, 0.224, 0.225])
# normalize=transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])
# data
train_dataset = dataset.SingleDataset(
cfg,
data_dir=cfg.DATA_DIR_TRAIN,
transform=transforms.Compose([
dataset.Resize((cfg.LOAD_SIZE, cfg.LOAD_SIZE)),
dataset.RandomCrop((cfg.FINE_SIZE, cfg.FINE_SIZE)),
dataset.RandomHorizontalFlip(),
dataset.ToTensor(),
dataset.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
]))
val_dataset = dataset.SingleDataset(
cfg,
data_dir=cfg.DATA_DIR_VAL,
transform=transforms.Compose([
dataset.Resize((cfg.LOAD_SIZE, cfg.LOAD_SIZE)),
dataset.CenterCrop((cfg.FINE_SIZE, cfg.FINE_SIZE)),
dataset.ToTensor(),
dataset.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
]))
# train_loader = torch.utils.data.DataLoader(
# train_dataset, batch_size=cfg.BATCH_SIZE, shuffle=True,
# num_workers=4, pin_memory=True, sampler=None)
# val_loader = torch.utils.data.DataLoader(
# val_dataset,batch_size=16, shuffle=False,
# num_workers=4, pin_memory=True)
train_loader = DataProvider(cfg,
dataset=train_dataset,
batch_size=40,
shuffle=True)
val_loader = DataProvider(cfg,
dataset=val_dataset,
batch_size=10,
shuffle=False)
# class weights
# num_classes_train = list(Counter([i[1] for i in train_loader.dataset.imgs]).values())
# cfg.CLASS_WEIGHTS_TRAIN = torch.FloatTensor(num_classes_train)
# writer = SummaryWriter(log_dir=cfg.LOG_PATH) # tensorboard
# net_classification_1=resnet50(pretrained=True)
# net_classification_2=resnet50(pretrained=True)
# net_classification_1 = models.__dict__['resnet18'](num_classes=365)
# net_classification_2 = models.__dict__['resnet18'](num_classes=365)
net_classification_1 = torch.hub.load('facebookresearch/WSL-Images',
'resnext101_32x16d_wsl')
# net_classification_2=torch.hub.load('facebookresearch/WSL-Images', 'resnext101_32x16d_wsl')
# for param in net_classification_1.parameters():
# param.requires_grad = False
# for param in net_classification_2.parameters():
# param.requires_grad = True
net_classification_1.fc = nn.Sequential(nn.Dropout(p=0.5),
nn.Linear(2048, 1024),
nn.LeakyReLU(inplace=True),
nn.Linear(1024, 67))
# net_classification_2.fc = nn.Sequential(nn.Dropout(p=0.5),nn.Linear(2048, 1024),nn.LeakyReLU(inplace=True),nn.Linear(1024,67))
net_classification_1.load_state_dict(
torch.load(
"./bestmodel/best_model_resnext_16d_2048_1024_dropout_0.5_b.pkl"))
# net_classification_2.load_state_dict(torch.load("./bestmodel/best_model_resnext_16d_2048_1024_dropout_0.5_b.pkl"))
# net_classification_2
# load_path = "/home/dudapeng/workspace/pretrained/place/resnet18_places365.pth"
# checkpoint = torch.load(load_path, map_location=lambda storage, loc: storage)
# state_dict = {str.replace(k, 'module.', ''): v for k, v in checkpoint['state_dict'].items()}
# net_classification_1.load_state_dict(state_dict)
# net_classification_2.load_state_dict(state_dict)
print(net_classification_1)
# num_ftrs = net_classification_1.fc.in_features
# net_classification_1.fc = nn.Linear(num_ftrs, cfg.NUM_CLASSES)
# num_ftrs = net_classification_2.fc.in_features
# net_classification_2.fc = nn.Linear(num_ftrs, cfg.NUM_CLASSES)
net_classification_1.cuda()
# net_classification_2.cuda()
cudnn.benchmark = True
# if cfg.GENERATE_Depth_DATA:
# print('GENERATE_Depth_DATA model set')
# cfg_generate = copy.deepcopy(cfg)
# cfg_generate.CHECKPOINTS_DIR='/home/lzy/generateDepth/checkpoints/best_AtoB/trecg_AtoB_best.pth'
# cfg_generate.GENERATE_Depth_DATA = False
# cfg_generate.NO_UPSAMPLE = False
# checkpoint = torch.load(cfg_generate.CHECKPOINTS_DIR)
# model = define_TrecgNet(cfg_generate, upsample=True,generate=True)
# load_checkpoint_depth(model,cfg_generate.CHECKPOINTS_DIR, checkpoint, data_para=True)
# generate_model = torch.nn.DataParallel(model).cuda()
# generate_model.eval()
net_classification_1 = torch.nn.DataParallel(net_classification_1).cuda()
# net_classification_2 = torch.nn.DataParallel(net_classification_2).cuda()
criterion = nn.CrossEntropyLoss().cuda()
best_mean_1 = 0
# best_mean_2=0
# optimizer = optim.SGD(model_ft.parameters(), lr=0.05,momentum=0.9)#,weight_decay=0.00005)
optimizer_1 = torch.optim.SGD(net_classification_1.parameters(),
lr=cfg.LR,
momentum=cfg.MOMENTUM,
weight_decay=cfg.WEIGHT_DECAY)
# optimizer_2 = torch.optim.SGD(net_classification_2.parameters(),lr=cfg.LR,momentum=cfg.MOMENTUM,weight_decay=cfg.WEIGHT_DECAY)
for epoch in range(0, 100):
adjust_learning_rate(optimizer_1, epoch)
# meanacc_1,meanacc_2=validate(val_loader, net_classification_1,net_classification_2,generate_model,criterion,epoch)
#
train(train_loader, net_classification_1, criterion, optimizer_1,
epoch, writer)
meanacc_1 = validate(val_loader, net_classification_1, criterion,
epoch, writer)
# meanacc_2=validate(val_loader,net_classification_2,generate_model,criterion,epoch,writer)
# train(train_loader,net_classification_2,generate_model,criterion,optimizer_2,epoch,writer)
# meanacc_2=validate(val_loader,net_classification_2,generate_model,criterion,epoch,writer)
# writer.add_image(depth_image[0])
# save best
if meanacc_1 > best_mean_1:
best_mean_1 = meanacc_1
print('best_mean_color:', str(best_mean_1))
save_checkpoint(
{
'epoch': epoch,
'arch': cfg.ARCH,
'state_dict': net_classification_1.state_dict(),
'best_mean_1': best_mean_1,
'optimizer': optimizer_1.state_dict(),
},
CorD=True)
# if meanacc_2>best_mean_2:
# best_mean_2=meanacc_2
# print('best_mean_depth:',str(best_mean_2))
# save_checkpoint({
# 'epoch': epoch,
# 'arch': cfg.ARCH,
# 'state_dict': net_classification_2.state_dict(),
# 'best_mean_2': best_mean_2,
# 'optimizer' : optimizer_2.state_dict(),
# },CorD=False)
print('best_mean_color:', str(best_mean_1))
writer.add_scalar('mean_acc_color', meanacc_1, global_step=epoch)
# writer.add_scalar('mean_acc_depth', meanacc_2, global_step=epoch)
writer.add_scalar('best_meanacc_color', best_mean_1, global_step=epoch)
# writer.add_scalar('best_meanacc_depth', best_mean_2, global_step=epoch)
writer.close()
os.environ['THEANO_FLAGS'] = 'device=gpu'
import os
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" # see issue #152
os.environ["CUDA_VISIBLE_DEVICES"] = "3"
config = tf.ConfigProto(log_device_placement=False, allow_soft_placement=True)
config.gpu_options.allow_growth = True
session = tf.Session(config=config)
K.set_session(session)
conf = Config(flag, args[2], int(args[3]))
print(flag)
# get data
dp = DataProvider(conf)
n_terms = len(dp.idx2word)
word_embed_data = np.array(dp.word_embed)
item_embed_data = np.random.rand(dp.get_item_size(), conf.dim_item)
word_transfer_W = np.random.rand(conf.dim_word, conf.dim_item)
word_transfer_b = np.random.rand(conf.dim_item)
print("finish data processing")
# define model
word_input = Input(shape=(1, ), dtype="int32", name="word_idx")
item_pos_input = Input(shape=(1, ), dtype="int32", name="item_pos_idx")
item_neg_input = Input(shape=(1, ), dtype="int32", name="item_neg_idx")
word_embed = Embedding(output_dim=conf.dim_word,
input_dim=n_terms,
def train(args):
height, width, channel = 28, 28, 1
batch_size = args.batch_size
z_size = args.nd # 噪声维数
real_img = tf.placeholder(tf.float32, [batch_size, height, width, channel],
name='img')
z = tf.placeholder(tf.float32, [batch_size, z_size], name='z')
label = tf.placeholder(tf.float32, [batch_size, 10], name='label') # 0~9
gan = model.GAN(height, width, channel)
gan.set_batch_size(batch_size)
fake_img = gan.generator(z, label)
real_result = gan.discriminator(real_img, label, reuse=False)
fake_result = gan.discriminator(fake_img, label, reuse=True)
real = tf.reduce_sum(label * real_result, 1)
fake = tf.reduce_sum(label * fake_result, 1)
d_loss = -tf.reduce_mean(tf.log(real) + tf.log(1. - fake))
g_loss = -tf.reduce_mean(tf.log(fake))
t_vars = tf.trainable_variables()
d_vars = [var for var in t_vars if 'dis' in var.name]
g_vars = [var for var in t_vars if 'gen' in var.name]
d_optimizer = tf.train.AdamOptimizer(learning_rate=0.0002, beta1=0.5) \
.minimize(d_loss, var_list=d_vars)
g_optimizer = tf.train.AdamOptimizer(learning_rate=0.0002, beta1=0.5) \
.minimize(g_loss, var_list=g_vars)
data = DataProvider()
train_num = data.get_train_num()
batch_num = int(train_num / args.batch_size)
saver = tf.train.Saver(max_to_keep=1)
model_dir = args.model_dir
if (not os.path.exists(model_dir)):
os.mkdir(model_dir)
accuracy_real = tf.reduce_mean(
tf.cast(tf.equal(tf.argmax(label, 1), tf.argmax(real_result, 1)),
'float'))
accuracy_fake = tf.reduce_mean(
tf.cast(tf.equal(tf.argmax(label, 1), tf.argmax(fake_result, 1)),
'float'))
with tf.Session() as sess:
counter = 0
sess.run(tf.global_variables_initializer())
for epoch in range(args.epoch):
for batch in range(batch_num):
counter += 1
train_data, label_data = data.next_batch(batch_size)
batch_z = np.random.normal(0, 1, [batch_size, z_size]).astype(
np.float_)
sess.run(d_optimizer,
feed_dict={
real_img: train_data,
z: batch_z,
label: label_data
})
sess.run(g_optimizer,
feed_dict={
z: batch_z,
label: label_data
})
if (counter % 20 == 0):
dloss, gloss, ac_real, ac_fake = sess.run(
[d_loss, g_loss, accuracy_real, accuracy_fake],
feed_dict={
real_img: train_data,
z: batch_z,
label: label_data
})
print('iter:', counter, 'd_loss:', dloss, 'g_loss:', gloss,
'ac_real:', ac_real, 'ac_fake:', ac_fake)
if (counter % 200 == 0):
saver.save(sess, os.path.join(model_dir, 'model'))
