def get_data_manager(self, root_dir=None, data_name='syn#1', seed_num=1):
'''
This function aims to construct a data manager instance to manage the
data info for the subsequent processing.
'''
data_info = data_name.split('#')
data_kind, data_num = data_info[0], int(data_info[1])
is_real = False if data_kind.startswith('syn') or data_kind.startswith(
'2d') else True
has_outliers = True if data_kind.endswith('otlr') else False
if data_kind == '2d':
self.cls_num, num_samples, num_features, dim_reduced = 3, 1000, 2, False
elif data_kind == 'tcga':
self.cls_num, num_samples, num_features, dim_reduced = 33, 11135, 5000, False
else:
self.cls_num, num_samples, num_features, dim_reduced = 10, 1000, 2000, False
data_manager = Data_Manager(root_dir = root_dir, is_real = is_real, data_kind = data_kind, \
data_num = data_num, has_outliers = has_outliers, dim_reduced = dim_reduced, \
num_of_features = num_features, num_of_samples = num_samples, num_of_cls = self.cls_num, \
seed = seed_num)
return data_manager
def export_fmu(model_path, file_name):
'''Parse signal exchange blocks and export boptest fmu and kpi json.
Parameters
----------
model_path : str
Path to orginal modelica model
file_name : list
Path(s) to modelica file and required libraries not on MODELICAPATH.
Passed to file_name parameter of pymodelica.compile_fmu() in JModelica.
Returns
-------
fmu_path : str
Path to the wrapped modelica model fmu
kpi_path : str
Path to kpi json
'''
# Get signal exchange instances and kpi signals
instances, signals = parse_instances(model_path, file_name)
# Write wrapper and export as fmu
fmu_path, _ = write_wrapper(model_path, file_name, instances)
# Write kpi json
kpi_path = os.path.join(os.getcwd(), 'kpis.json')
with open(kpi_path, 'w') as f:
json.dump(signals, f)
# open up the FMU and save the kpis json
man = Data_Manager()
man.save_data_and_kpisjson(fmu_path=fmu_path)
return fmu_path, kpi_path
def load_checkpoint(dir_name, lr, HAS_CUDA):
# load model parameters
if HAS_CUDA:
device = torch.cuda.current_device()
print(f'Cuda device: {device}')
checkpoint = torch.load(
dir_name + '/' + 'checkpoint.pth.tar',
map_location=lambda storage, loc: storage.cuda(device))
print('Loaded CUDA version')
else:
checkpoint = torch.load(dir_name + '/' + 'checkpoint.pth.tar',
map_location='cpu')
# load pretrained model
pt_model = checkpoint['pt_model']
model_pt = models.__dict__[pt_model](pretrained=True)
# Recreate model
img_cl = Composite_Classifier(model_pt, checkpoint['n_hid'],
checkpoint['drops'], checkpoint['num_cat'])
# load model state disctionary
img_cl.load_state_dict(checkpoint['model'])
# Recreate optimiser
optimizer_ft = optim.SGD(img_cl.cf_layers.parameters(),
lr=0.001,
momentum=0.9)
optimizer_ft.load_state_dict(checkpoint['optimizer'])
old_lr = optimizer_ft.param_groups[0]['lr']
last_epoch_trained_upon = checkpoint['epochs']
exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft,
step_size=7,
gamma=0.1)
# Now move optimise to GPU if necessary
if HAS_CUDA:
for state in optimizer_ft.state.values():
for k, v in state.items():
if isinstance(v, torch.Tensor):
state[k] = v.to(device)
if old_lr != lr:
optimizer_ft.param_groups[0]['lr'] = lr
else:
# if lr has not been updated put scheduler back to where it was
exp_lr_scheduler.last_epoch = last_epoch_trained_upon
# Recreate data object
data = Data_Manager(checkpoint['data_dir'], checkpoint['phases'],
checkpoint['data_tfms'], checkpoint['bs'])
# Recreate model manager class instance
phases = checkpoint['phases']
model_mgr = Solution_Manager(img_cl, checkpoint['loss_function'],
optimizer_ft, exp_lr_scheduler, data, phases,
HAS_CUDA)
# restore model manager state variables
model_mgr.epochs = checkpoint['epochs']
model_mgr.loss_function = checkpoint['loss_function']
model_mgr.best_accuracy = checkpoint['best_accuracy']
model_mgr.best_corrects = checkpoint['best_corrects']
model_mgr.best_loss = checkpoint['best_loss']
model_mgr.model.class_to_idx = checkpoint['class_to_idx']
if HAS_CUDA:
model_mgr.model.cuda()
# Freeze the pre-trained model layers
for param in img_cl.model_pt.parameters():
param.requires_grad = False
print('Checkpoint loaded')
return model_mgr, pt_model
def main():
global args
args = parser.parse_args()
# Default settings
default_model = 'resnet18'
default_bs = 16
sz = 224
# Take actions based upon initial arguments
if args.gpu:
# Check for GPU and CUDA libraries
HAS_CUDA = torch.cuda.is_available()
if not HAS_CUDA:
sys.exit('No Cuda capable GPU detected')
else:
HAS_CUDA = False
checkpoint_dir = args.save_dir
# Define hyper-parameters
# Note - allow dropout to be changed when resuming model
tmp = args.dropout
tmp = re.sub("[\[\]]", "", tmp)
drops = [float(item) for item in tmp.split(',')]
lr = args.learning_rate
epochs = args.epochs
# All arguments imported, will start to setup model depending upon whether restarting from checkpoint or
# from scratch
if args.resume:
if os.path.isdir(args.resume):
print('Loading checkpoint...')
sol_mgr, pt_model = utility.load_checkpoint(
args.resume, lr, HAS_CUDA)
else:
# Define hidden layer details (note - if resuming will continue with values used earlier
tmp = args.hidden_units
tmp = re.sub("[\[\]]", "", tmp)
n_hid = [int(item) for item in tmp.split(',')]
# check data directory exists and assign
data_dir = args.data_directory
# Check it exists
if not os.path.exists(data_dir):
sys.exit('Data directory does not exist')
# Create model, datasets etc from scratch
# create datasets and dataloaders
phrases = ['train', 'valid', 'test']
# Define data transforms
data_transforms = {
'train':
transforms.Compose([
transforms.RandomResizedCrop(sz),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406],
[0.229, 0.224, 0.225])
]),
'valid':
transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(sz),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406],
[0.229, 0.224, 0.225])
]),
'test':
transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(sz),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406],
[0.229, 0.224, 0.225])
]),
}
bs = args.batch_size
data = Data_Manager(data_dir, phrases, data_transforms, bs)
# Load cat_to_name
cat_to_name = utility.load_classes('cat_to_name.json')
num_cat = len(cat_to_name)
# Load pre-trained model
if args.arch is not None:
pt_model = args.arch
else:
pt_model = default_model
model_pt = models.__dict__[pt_model](pretrained=True)
num_ftrs = model_pt.fc.in_features
# Create classifier model
img_cl = Composite_Classifier(model_pt, n_hid, drops, num_cat)
# Move to CUDA if available
if HAS_CUDA:
img_cl.cuda()
# Define losses and hyper-parameters
criterion = nn.CrossEntropyLoss()
# Optimise just the parameters of the classifier layers
optimizer_ft = optim.SGD(img_cl.cf_layers.parameters(),
lr=lr,
momentum=0.9)
exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft,
step_size=7,
gamma=0.1)
# Freeze the pre-trained model layers
for param in img_cl.model_pt.parameters():
param.requires_grad = False
# Create model manager to control training, validation, test and predict with the model and data
sol_mgr = Solution_Manager(img_cl,
criterion,
optimizer_ft,
exp_lr_scheduler,
data,
phrases,
HAS_CUDA=HAS_CUDA)
sol_mgr.model.class_to_idx = data.image_datasets['train'].class_to_idx
# Train model
sol_mgr.train(epochs=epochs)
# Evaluate model against test set
sol_mgr.test_with_dl()
# Save Checkpoint
utility.save_checkpoint(args.save_dir, sol_mgr, pt_model, HAS_CUDA)
def __init__(self,
board_dim=8,
time_steps=1,
n_filters=256,
conv_size=3,
n_res=40,
c=.1):
"""
:param board_dim: dimension of game board
:param time_steps: number of time steps kept in state history
:param n_filters: number of convolutional filters per conv layer
:param conv_size: size of convolutions
:param n_res: number of residual layers
:param c: regularization scale constant
"""
self.board_dim = board_dim
self.time_steps = time_steps
self.losses = None
self.n_conv_filters = n_filters
self.conv_size = conv_size
self.n_res_layers = n_res
self.regularizer = tf.contrib.layers.l2_regularizer(scale=c)
self.dm = Data_Manager(max_size=(board_dim**2 - 4) *
500) # moves per game TIMES num games to save
# --------------
# Make Network
# --------------
with tf.Graph().as_default() as net1_graph:
self.input_layer = tf.placeholder(shape=[
None, self.board_dim, self.board_dim, (self.time_steps * 2 + 1)
],
dtype=tf.float32,
name='input')
self.net = self._add_conv_layer(self.input_layer, name='conv1')
for i in range(self.n_res_layers):
self.net = self._add_res_layer(self.net,
name='res{}'.format(i + 1))
self.policy_logits = self._policy_head(self.net)
self.value_estimate = self._value_head(self.net)
self.mcts_pi = tf.placeholder(
shape=[None, (self.board_dim**2 + 1)],
dtype=tf.float32,
name='pi')
self.winner_z = tf.placeholder(shape=[None, 1],
dtype=tf.float32,
name='z')
# Loss, composed of cross entropy, mse, and regularization
xent = tf.nn.softmax_cross_entropy_with_logits(
labels=self.mcts_pi, logits=self.policy_logits)
mse = tf.losses.mean_squared_error(self.winner_z,
self.value_estimate)
reg_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
self.loss = tf.reduce_mean(mse - xent + sum(reg_losses))
self.optimizer = tf.train.AdamOptimizer().minimize(
self.loss) # tune learning rate
# more ops
self.init_op = tf.global_variables_initializer()
self.saver = tf.train.Saver()
# initialize session
self.sess = tf.Session(graph=net1_graph)
self.sess.run(self.init_op)
from agent import Agent
visualizer.init_visualizer()
WINDOW_SIZE = 60
BATCH_SIZE = 30
EPISODE = 8
LEARNING_RATE = 0.001
VALIDATION = 0 # train data에서 이 비율만큼 validation data로 사용
ENSEMBLE_NUM = 16
USE_TOP_N_AGENT = ENSEMBLE_NUM // 3
ROLLING_TRAIN_TEST = False
# 학습/ 테스트 data 설정
dm = Data_Manager('./gaps.db', 20151113, 20180615, split_ratio=(0.6, 0.2, 0.2))
df = dm.load_db()
train_df, val_df, test_df = dm.generate_feature_df(df, WINDOW_SIZE)
print('train: {} ~ {}'.format(train_df.iloc[0].name, train_df.iloc[-1].name))
print('val : {} ~ {}'.format(val_df.iloc[WINDOW_SIZE].name,
val_df.iloc[-1].name))
print('test: {} ~ {}'.format(test_df.iloc[WINDOW_SIZE].name,
test_df.iloc[-1].name))
print("데이터 수 train: {}, val: {}, test: {}".format(len(train_df), len(val_df),
len(test_df)))
visualizer.plot_dfs(
[train_df, val_df.iloc[WINDOW_SIZE:], test_df.iloc[WINDOW_SIZE:]],
['train', 'val', 'test'])
print("학습 데이터의 asset 개수 : ", len(train_df.columns.levels[0]))
