def __init__(self, args):
self.args = args
self.cfg = load_config(args.cfg_name)
self.converter = LabelConverter(chars_file=args.chars_file)
self.tr_ds = Dataset(self.cfg, args.train_dir, args.train_gt_dir,
self.converter, self.cfg.batch_size)
self.cfg.lr_boundaries = [self.tr_ds.num_batches * epoch for epoch in self.cfg.lr_decay_epochs]
self.cfg.lr_values = [self.cfg.lr * (self.cfg.lr_decay_rate ** i) for i in
range(len(self.cfg.lr_boundaries) + 1)]
if args.val_dir is None:
self.val_ds = None
else:
self.val_ds = Dataset(self.cfg, args.val_dir, args.val_gt_dir,
self.converter, self.cfg.batch_size, shuffle=False)
if args.test_dir is None:
self.test_ds = None
else:
# Test images often have different size, so set batch_size to 1
self.test_ds = Dataset(self.cfg, args.test_dir, args.test_gt_dir,
self.converter, shuffle=False, batch_size=1)
self.model = ResNetV2(self.cfg, self.converter.num_classes)
self.model.create_architecture()
self.sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
self.epoch_start_index = 0
self.batch_start_index = 0
def __init__(self, default_params, specific_params, db_client, exchange_client=None, is_backtest=False):
# initialize status and settings of bot.
# if you try backtest, db_client is in need.
self.exchange_client = exchange_client
self.db_client = db_client
self.default_params = default_params
self.extract_default_params(self.default_params)
self.specific_params = specific_params
self.combined_params = dict(**self.default_params, **self.specific_params)
self.is_backtest = is_backtest
if is_backtest:
# for params table
self.backtest_management_table_name = self.bot_name + "_backtest_management"
# backtest configure
self.initial_balance = 100.0 # USD
self.account_currency = "USD"
self.dataset_manipulator = Dataset(self.db_client, self.exchange_client)
if self.db_client.is_table_exist(self.backtest_management_table_name) is not True:
self.create_backtest_management_table()
if self.db_client.is_table_exist("backtest_management") is True:
# delete useless template table
drop_query = "DROP TABLE backtest_management;"
self.db_client.exec_sql(drop_query, return_df=False)
def main():
parser = argparse.ArgumentParser(description="Import dataset into a useful representation that we can use")
parser.add_argument("dataset_file", help="the file to import")
parser.add_argument("data_directory", help="the location where we are storing the data.")
args = parser.parse_args()
print("processing file: " + args.dataset_file)
if not path.exists(args.dataset_file):
raise Exception("no such file: " + args.dataset_file)
dataset_file, data_directory = open_file(args.dataset_file), args.data_directory
ts_remove_str_len = -len(".000Z")
with Dataset(data_directory) as dataset:
for line in tqdm(dataset_file):
try:
if type(line) != str:
line = line.decode("ascii")
if len(line) == 0: continue
_, az, insttype, os, price, timestamp_str = tuple(line.strip().split("\t"))
price = float(price)
timestamp = datetime.strptime(timestamp_str[:ts_remove_str_len], "%Y-%m-%dT%H:%M:%S")
dataset.insert_data(az, insttype, time_to_epoch(timestamp), price)
except Exception as e:
print(e)
def make_match_json(json_path, dst_dir, num=4):
"""
prepare the top 4 match for each id
store the dataset in dst_dir and write the results into json
"""
dataset = Dataset()
dataset.make_data(json_path)
pickle.dump(dataset, open(dst_dir + 'dataset.pkl', 'wb'))
dic = {}
for person in dataset.people:
for i in range(person.img_num):
match = euclid_match_human(person, i, num)
dic[person.imgs[i]] = match
f = open(dst_dir + 'match.json', 'w')
json.dump(dic, f)
def __init__(self, config, device, sess=None, graph=None):
'''
if game_lm is not None, the result (mesh obj and UV texture map)
will be convert from nsh to the game
'''
self.config = config
self.name = config.name
self.device = device
self.sess = sess
self.graph = graph
self.log = logging.getLogger('x')
self.rot_order = 'XYZ'
self.debug = config.debug
self.ex_idx = [4, 5, 8]
self.inpaint_model = InpaintingModel(config,
device,
self.rot_order,
debug=self.debug).to(device)
# self.inpaint_model = InpaintingModel(config, device, self.debug)
self.epoch = 0
if config.restore:
self.epoch = self.inpaint_model.load()
# self.phase = config.phase
if config.mode == 'train':
num_test = 2048
flist = glob(os.path.join(config.data_dir, '*_uv.png'))
random.shuffle(flist)
train_flist = flist[:-2 * num_test]
val_flist = flist[-2 * num_test:-num_test]
test_flist = flist[-num_test:]
num_test = 300
flist_gt = glob(os.path.join(config.data_gt_dir, '*_uv*.png'))
random.shuffle(flist_gt)
train_flist_gt = flist_gt[:-2 * num_test]
val_flist_gt = flist_gt[-2 * num_test:-num_test]
test_flist_gt = flist_gt[-num_test:]
self.train_dataset = Dataset(config, train_flist_gt, train_flist)
self.val_dataset = Dataset(config, val_flist_gt, val_flist)
self.val_sample_iterator = self.val_dataset.create_iterator(
config.batch_size)
self.test_dataset = Dataset(config,
test_flist_gt,
test_flist,
test=True)
self.test_sample_iterator = self.test_dataset.create_iterator(
config.batch_size)
self.samples_dir = os.path.join('samples', config.name)
os.makedirs(self.samples_dir, exist_ok=True)
elif config.mode == 'test':
self.test_dataset = Dataset(config, [], [], test=True)
self.init_test()
def set_helper_libs(self):
self.dataset_manipulator = Dataset(self.db_client,
self.exchange_client,
self.is_backtest)
self.ohlcv_tradingbot = OHLCV_tradingbot(self.dataset_manipulator,
self.default_params,
self.specific_params)
if self.is_backtest:
self.trading_bot_backtest = TradingBotBacktest(self)
else:
self.trading_bot_real = TradingBotReal(self)
self.line = LineNotification(self.db_client.config_path)
def collect_predictions(
model: Union[str, keras.models.Model],
test_gen: dataset.Dataset) -> Tuple[np.ndarray, np.ndarray]:
""" Loads a model and collects predictions.
Args:
model: a Keras model or a path to a Keras model.
test_gen: generator for test data.
Returns:
A tuple of predictions and true values.
"""
if isinstance(model, keras.models.Model):
model = keras.models.load_model(model)
pred, true = np.empty((0, 7)), np.empty((0, 7))
for i in tqdm.tqdm(range(1000)):
data = test_gen.generate()
pred = np.append(pred, model.predict(data[0]), axis=0)
true = np.append(true, data[1], axis=0)
return pred, true
class Trainer(object):
def __init__(self, args):
self.args = args
self.cfg = load_config(args.cfg_name)
self.converter = LabelConverter(chars_file=args.chars_file)
self.tr_ds = Dataset(self.cfg, args.train_dir, args.train_gt_dir,
self.converter, self.cfg.batch_size)
self.cfg.lr_boundaries = [self.tr_ds.num_batches * epoch for epoch in self.cfg.lr_decay_epochs]
self.cfg.lr_values = [self.cfg.lr * (self.cfg.lr_decay_rate ** i) for i in
range(len(self.cfg.lr_boundaries) + 1)]
if args.val_dir is None:
self.val_ds = None
else:
self.val_ds = Dataset(self.cfg, args.val_dir, args.val_gt_dir,
self.converter, self.cfg.batch_size, shuffle=False)
if args.test_dir is None:
self.test_ds = None
else:
# Test images often have different size, so set batch_size to 1
self.test_ds = Dataset(self.cfg, args.test_dir, args.test_gt_dir,
self.converter, shuffle=False, batch_size=1)
self.model = ResNetV2(self.cfg, self.converter.num_classes)
self.model.create_architecture()
self.sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
self.epoch_start_index = 0
self.batch_start_index = 0
def train(self):
self.sess.run(tf.global_variables_initializer())
self.saver = tf.train.Saver(tf.global_variables(), max_to_keep=8)
self.train_writer = tf.summary.FileWriter(self.args.log_dir, self.sess.graph)
# if self.args.restore:
# self._restore()
print('Begin training...')
for epoch in range(self.epoch_start_index, self.cfg.epochs):
self.sess.run(self.tr_ds.init_op)
for batch in range(self.batch_start_index, self.tr_ds.num_batches):
batch_start_time = time.time()
total_cost, detect_loss, detect_cls_loss, detect_reg_loss, reco_loss, global_step, lr = self._train()
# if batch != 0 and (batch % self.args.log_step == 0):
# batch_cost, global_step, lr = self._train_with_summary()
# else:
# batch_cost, global_step, lr = self._train()
print("{:.02f}s, epoch: {}, batch: {}/{}, total_loss: {:.03}, reco_loss: {:.03}, "
"detect_loss (total: {:.03}, cls: {:.03}, reg: {:.03}), "
"lr: {:.05}"
.format(time.time() - batch_start_time, epoch, batch, self.tr_ds.num_batches,
total_cost, reco_loss, detect_loss, detect_cls_loss, detect_reg_loss, lr))
# if global_step != 0 and (global_step % self.args.val_step == 0):
# val_acc = self._do_val(self.val_ds, epoch, global_step, "val")
# test_acc = self._do_val(self.test_ds, epoch, global_step, "test")
# self._save_checkpoint(self.args.ckpt_dir, global_step, val_acc, test_acc)
self.batch_start_index = 0
# def _restore(self):
# utils.restore_ckpt(self.sess, self.saver, self.args.ckpt_dir)
#
# step_restored = self.sess.run(self.model.global_step)
#
# self.epoch_start_index = math.floor(step_restored / self.tr_ds.num_batches)
# self.batch_start_index = step_restored % self.tr_ds.num_batches
#
# print("Restored global step: %d" % step_restored)
# print("Restored epoch: %d" % self.epoch_start_index)
# print("Restored batch_start_index: %d" % self.batch_start_index)
def _train(self):
imgs, score_maps, geo_maps, text_roi_count, affine_matrixs, affine_rects, labels, img_paths = \
self.tr_ds.get_next_batch(self.sess)
# print(imgs.shape)
# print(score_maps.shape)
# print(geo_maps.shape)
# print(affine_matrixs.shape)
# print(affine_rects.shape)
# print(labels[0].shape)
fetches = [
self.model.total_loss,
self.model.detect_loss,
self.model.detect_cls_loss,
self.model.detect_reg_loss,
self.model.reco_ctc_loss,
self.model.global_step,
self.model.lr,
self.model.train_op
]
feed = {
self.model.input_images: imgs,
self.model.input_score_maps: score_maps,
self.model.input_geo_maps: geo_maps,
self.model.input_text_roi_count: text_roi_count,
self.model.input_affine_matrixs: affine_matrixs,
self.model.input_affine_rects: affine_rects,
self.model.input_text_labels: labels,
self.model.is_training: True
}
# try:
total_loss, detect_loss, detect_cls_loss, detect_reg_loss, reco_ctc_loss, global_step, lr, _ = self.sess.run(
fetches, feed)
# except:
# print(img_paths)
# exit(-1)
return total_loss, detect_loss, detect_cls_loss, detect_reg_loss, reco_ctc_loss, global_step, lr
# def _train_with_summary(self):
# img_batch, label_batch, labels, _ = self.tr_ds.get_next_batch(self.sess)
# feed = {self.model.inputs: img_batch,
# self.model.labels: label_batch,
# self.model.is_training: True}
#
# fetches = [self.model.total_loss,
# self.model.ctc_loss,
# self.model.regularization_loss,
# self.model.global_step,
# self.model.lr,
# self.model.merged_summay,
# self.model.dense_decoded,
# self.model.edit_distance,
# self.model.train_op]
#
# batch_cost, _, _, global_step, lr, summary, predicts, edit_distance, _ = self.sess.run(fetches, feed)
# self.train_writer.add_summary(summary, global_step)
#
# predicts = [self.converter.decode(p, CRNN.CTC_INVALID_INDEX) for p in predicts]
# accuracy, _ = infer.calculate_accuracy(predicts, labels)
#
# tf_utils.add_scalar_summary(self.train_writer, "train_accuracy", accuracy, global_step)
# tf_utils.add_scalar_summary(self.train_writer, "train_edit_distance", edit_distance, global_step)
#
# return batch_cost, global_step, lr
# def _do_val(self, dataset, epoch, step, name):
# if dataset is None:
# return None
#
# accuracy, edit_distance = infer.validation(self.sess, self.model.feeds(), self.model.fetches(),
# dataset, self.converter, self.args.result_dir, name, step)
#
# tf_utils.add_scalar_summary(self.train_writer, "%s_accuracy" % name, accuracy, step)
# tf_utils.add_scalar_summary(self.train_writer, "%s_edit_distance" % name, edit_distance, step)
#
# print("epoch: %d/%d, %s accuracy = %.3f" % (epoch, self.cfg.epochs, name, accuracy))
# return accuracy
def _save_checkpoint(self, ckpt_dir, step, val_acc=None, test_acc=None):
ckpt_name = "crnn_%d" % step
if val_acc is not None:
ckpt_name += '_val_%.03f' % val_acc
if test_acc is not None:
ckpt_name += '_test_%.03f' % test_acc
name = os.path.join(ckpt_dir, ckpt_name)
print("save checkpoint %s" % name)
meta_exists, meta_file_name = self._meta_file_exist(ckpt_dir)
self.saver.save(self.sess, name)
# remove old meta file to save disk space
if meta_exists:
try:
os.remove(os.path.join(ckpt_dir, meta_file_name))
except:
print('Remove meta file failed: %s' % meta_file_name)
def _meta_file_exist(self, ckpt_dir):
fnames = os.listdir(ckpt_dir)
meta_exists = False
meta_file_name = ''
for n in fnames:
if 'meta' in n:
meta_exists = True
meta_file_name = n
break
return meta_exists, meta_file_name
def main():
logger = None
output_dir = ''
setup_default_logging()
args = parser.parse_args()
args.prefetcher = not args.no_prefetcher
args.distributed = False
if 'WORLD_SIZE' in os.environ:
args.distributed = int(os.environ['WORLD_SIZE']) > 1
if args.distributed and args.num_gpu > 1:
logging.warning(
'Using more than one GPU per process in distributed mode is not allowed. Setting num_gpu to 1.'
)
args.num_gpu = 1
args.device = 'cuda:0'
args.world_size = 1
args.rank = 0 # global rank
if args.distributed:
args.num_gpu = 1
args.device = 'cuda:%d' % args.local_rank
torch.cuda.set_device(args.local_rank)
import random
port = random.randint(0, 50000)
torch.distributed.init_process_group(
backend='nccl', init_method='env://'
) # tcp://127.0.0.1:{}'.format(port), rank=args.local_rank, world_size=8)
args.world_size = torch.distributed.get_world_size()
args.rank = torch.distributed.get_rank()
assert args.rank >= 0
if args.distributed:
logging.info(
'Training in distributed mode with multiple processes, 1 GPU per process. Process %d, total %d.'
% (args.rank, args.world_size))
else:
logging.info('Training with a single process on %d GPUs.' %
args.num_gpu)
seed = args.seed
torch.manual_seed(seed)
torch.cuda.manual_seed_all(seed)
np.random.seed(seed)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
model, sta_num, size_factor = _gen_supernet(
flops_minimum=args.flops_minimum,
flops_maximum=args.flops_maximum,
num_classes=args.num_classes,
drop_rate=args.drop,
global_pool=args.gp,
resunit=args.resunit,
dil_conv=args.dil_conv,
slice=args.slice)
if args.local_rank == 0:
print("Model Searched Using FLOPs {}".format(size_factor * 32))
data_config = resolve_data_config(vars(args),
model=model,
verbose=args.local_rank == 0)
if args.local_rank == 0:
output_base = args.output if args.output else './experiments'
exp_name = '-'.join([
datetime.now().strftime("%Y%m%d-%H%M%S"), args.model,
str(data_config['input_size'][-1]),
str(args.flops_minimum),
str(args.flops_maximum)
])
output_dir = get_outdir(output_base, 'search', exp_name)
log_file = os.path.join(output_dir, "search.log")
logger = get_logger(log_file)
if args.local_rank == 0:
logger.info(args)
choice_num = 6
if args.resunit:
choice_num += 1
if args.dil_conv:
choice_num += 2
if args.local_rank == 0:
logger.info("Choice_num: {}".format(choice_num))
model_est = LatencyEst(model)
if os.path.exists(args.initial_checkpoint):
load_checkpoint(model, args.initial_checkpoint)
if args.local_rank == 0:
logger.info('Model %s created, param count: %d' %
(args.model, sum([m.numel() for m in model.parameters()])))
# data_config = resolve_data_config(vars(args), model=model, verbose=args.local_rank == 0)
# optionally resume from a checkpoint
optimizer_state = None
resume_epoch = None
if args.resume:
optimizer_state, resume_epoch = resume_checkpoint(model, args.resume)
if args.num_gpu > 1:
if args.amp:
logging.warning(
'AMP does not work well with nn.DataParallel, disabling. Use distributed mode for multi-GPU AMP.'
)
args.amp = False
model = nn.DataParallel(model,
device_ids=list(range(args.num_gpu))).cuda()
else:
model.cuda()
optimizer = create_optimizer_supernet(args, model)
if optimizer_state is not None:
optimizer.load_state_dict(optimizer_state['optimizer'])
use_amp = False
if has_apex and args.amp:
model, optimizer = amp.initialize(model, optimizer, opt_level='O1')
use_amp = True
if args.local_rank == 0:
logger.info('NVIDIA APEX {}. AMP {}.'.format(
'installed' if has_apex else 'not installed',
'on' if use_amp else 'off'))
model_ema = None
if args.model_ema:
# Important to create EMA model after cuda(), DP wrapper, and AMP but before SyncBN and DDP wrapper
model_ema = ModelEma(model,
decay=args.model_ema_decay,
device='cpu' if args.model_ema_force_cpu else '',
resume=args.resume)
if args.distributed:
if args.sync_bn:
try:
if has_apex:
model = convert_syncbn_model(model)
else:
model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(
model)
if args.local_rank == 0:
logger.info(
'Converted model to use Synchronized BatchNorm.')
except Exception as e:
logging.error(
'Failed to enable Synchronized BatchNorm. Install Apex or Torch >= 1.1'
)
if has_apex:
model = DDP(model, delay_allreduce=True)
else:
if args.local_rank == 0:
logger.info(
"Using torch DistributedDataParallel. Install NVIDIA Apex for Apex DDP."
)
model = DDP(model,
device_ids=[args.local_rank
]) # can use device str in Torch >= 1.1
# NOTE: EMA model does not need to be wrapped by DDP
lr_scheduler, num_epochs = create_scheduler(args, optimizer)
start_epoch = 0
if args.start_epoch is not None:
# a specified start_epoch will always override the resume epoch
start_epoch = args.start_epoch
elif resume_epoch is not None:
start_epoch = resume_epoch
if start_epoch > 0:
lr_scheduler.step(start_epoch)
if args.local_rank == 0:
logger.info('Scheduled epochs: {}'.format(num_epochs))
if args.tiny:
from lib.dataset.tiny_imagenet import get_newimagenet
[loader_train,
loader_eval], [train_sampler, test_sampler
] = get_newimagenet(args.data, args.batch_size)
else:
train_dir = os.path.join(args.data, 'train')
if not os.path.exists(train_dir):
logger.error(
'Training folder does not exist at: {}'.format(train_dir))
exit(1)
dataset_train = Dataset(train_dir)
collate_fn = None
loader_train = create_loader(
dataset_train,
input_size=data_config['input_size'],
batch_size=args.batch_size,
is_training=True,
use_prefetcher=args.prefetcher,
re_prob=args.reprob,
re_mode=args.remode,
color_jitter=args.color_jitter,
interpolation=
'random', # FIXME cleanly resolve this? data_config['interpolation'],
mean=data_config['mean'],
std=data_config['std'],
num_workers=args.workers,
distributed=args.distributed,
collate_fn=collate_fn,
)
eval_dir = os.path.join(args.data, 'val')
if not os.path.isdir(eval_dir):
logger.error(
'Validation folder does not exist at: {}'.format(eval_dir))
exit(1)
dataset_eval = Dataset(eval_dir)
loader_eval = create_loader(
dataset_eval,
input_size=data_config['input_size'],
batch_size=4 * args.batch_size,
is_training=False,
use_prefetcher=args.prefetcher,
interpolation=data_config['interpolation'],
mean=data_config['mean'],
std=data_config['std'],
num_workers=args.workers,
distributed=args.distributed,
)
if args.smoothing:
train_loss_fn = LabelSmoothingCrossEntropy(
smoothing=args.smoothing).cuda()
validate_loss_fn = nn.CrossEntropyLoss().cuda()
else:
train_loss_fn = nn.CrossEntropyLoss().cuda()
validate_loss_fn = train_loss_fn
eval_metric = args.eval_metric
best_metric = None
best_epoch = None
saver = None
best_children_pool = []
if args.local_rank == 0:
decreasing = True if eval_metric == 'loss' else False
saver = CheckpointSaver(checkpoint_dir=output_dir,
decreasing=decreasing)
try:
for epoch in range(start_epoch, num_epochs):
if args.distributed:
if args.tiny:
train_sampler.set_epoch(epoch)
else:
loader_train.sampler.set_epoch(epoch)
train_metrics, best_children_pool = train_epoch(
epoch,
model,
loader_train,
optimizer,
train_loss_fn,
args,
CHOICE_NUM=choice_num,
lr_scheduler=lr_scheduler,
saver=saver,
output_dir=output_dir,
logger=logger,
val_loader=loader_eval,
best_children_pool=best_children_pool,
use_amp=use_amp,
model_ema=model_ema,
est=model_est,
sta_num=sta_num)
# eval_metrics = OrderedDict([('loss', 0.0), ('prec1', 0.0), ('prec5', 0.0)])
eval_metrics = validate(model,
loader_eval,
validate_loss_fn,
args,
CHOICE_NUM=choice_num,
sta_num=sta_num)
update_summary(epoch,
train_metrics,
eval_metrics,
os.path.join(output_dir, 'summary.csv'),
write_header=best_metric is None)
if saver is not None:
# save proper checkpoint with eval metric
save_metric = eval_metrics[eval_metric]
best_metric, best_epoch = saver.save_checkpoint(
model,
optimizer,
args,
epoch=epoch,
model_ema=model_ema,
metric=save_metric)
except KeyboardInterrupt:
pass
if best_metric is not None:
logging.info('*** Best metric: {0} (epoch {1})'.format(
best_metric, best_epoch))
def main():
test_args = parse_args()
args = joblib.load('models/%s/args.pkl' % test_args.name)
print('Config -----')
for arg in vars(args):
print('%s: %s' % (arg, getattr(args, arg)))
print('------------')
if args.pred_type == 'classification':
num_outputs = 5
elif args.pred_type == 'regression':
num_outputs = 1
elif args.pred_type == 'multitask':
num_outputs = 6
else:
raise NotImplementedError
cudnn.benchmark = True
test_transform = transforms.Compose([
transforms.Resize((args.input_size)),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
])
# data loading code
test_dir = preprocess('test',
args.img_size,
scale=args.scale_radius,
norm=args.normalize,
pad=args.padding,
remove=args.remove)
test_df = pd.read_csv('inputs/test.csv')
test_img_paths = test_dir + '/' + test_df['id_code'].values + '.png'
test_labels = np.zeros(len(test_img_paths))
test_set = Dataset(test_img_paths, test_labels, transform=test_transform)
test_loader = torch.utils.data.DataLoader(test_set,
batch_size=args.batch_size,
shuffle=False,
num_workers=4)
preds = []
for fold in range(args.n_splits):
print('Fold [%d/%d]' % (fold + 1, args.n_splits))
# create model
model_path = 'models/%s/model_%d.pth' % (args.name, fold + 1)
if not os.path.exists(model_path):
print('%s is not exists.' % model_path)
continue
model = get_model(model_name=args.arch,
num_outputs=num_outputs,
freeze_bn=args.freeze_bn,
dropout_p=args.dropout_p)
model = model.cuda()
model.load_state_dict(torch.load(model_path))
model.eval()
preds_fold = []
with torch.no_grad():
for i, (input, _) in tqdm(enumerate(test_loader),
total=len(test_loader)):
if test_args.tta:
outputs = []
for input in apply_tta(input):
input = input.cuda()
output = model(input)
outputs.append(output.data.cpu().numpy()[:, 0])
preds_fold.extend(np.mean(outputs, axis=0))
else:
input = input.cuda()
output = model(input)
preds_fold.extend(output.data.cpu().numpy()[:, 0])
preds_fold = np.array(preds_fold)
preds.append(preds_fold)
if not args.cv:
break
preds = np.mean(preds, axis=0)
if test_args.tta:
args.name += '_tta'
test_df['diagnosis'] = preds
test_df.to_csv('probs/%s.csv' % args.name, index=False)
thrs = [0.5, 1.5, 2.5, 3.5]
preds[preds < thrs[0]] = 0
preds[(preds >= thrs[0]) & (preds < thrs[1])] = 1
preds[(preds >= thrs[1]) & (preds < thrs[2])] = 2
preds[(preds >= thrs[2]) & (preds < thrs[3])] = 3
preds[preds >= thrs[3]] = 4
preds = preds.astype('int')
test_df['diagnosis'] = preds
test_df.to_csv('submissions/%s.csv' % args.name, index=False)
# Training/Testing
if settings.phase == 'training':
# Find patch locations or load from cache.
patch_locations_path = join(settings.out_cache_path, 'patch_locations.pkl')
if settings.find_patch_locations or not isfile(patch_locations_path):
LOGGER.info('Calculating patch locations ...')
find_and_store_patch_locations(settings)
with open(patch_locations_path, 'rb') as handle:
patch_locations = pickle.load(handle)
LOGGER.info('Patch locations loaded ...')
# Dataset iterators.
LOGGER.info('Initializing training and validation datasets ...')
training_dataset = Dataset(settings, patch_locations, phase='train')
validation_dataset = Dataset(settings, patch_locations, phase='val')
if settings.eval:
model.restore_model('checkpoints-40000')
validation_dataset = Dataset(settings, patch_locations, phase='val')
error = model.run_inference_val(validation_dataset)
LOGGER.info('Validation 3 pixel error: {}'.format(error))
else:
# Training.
LOGGER.info('Starting training ...')
model.fit(training_dataset, validation_dataset, optimizer,
settings.num_iterations)
LOGGER.info('Training done ...')
elif settings.phase == 'testing':
model.restore_model('checkpoints-40000')
def main():
args = parse_args()
if args.name is None:
args.name = '%s_%s' % (args.mode, args.arch)
if not os.path.exists('models/%s' % args.name):
os.makedirs('models/%s' % args.name)
print('Config -----')
for arg in vars(args):
print('- %s: %s' % (arg, getattr(args, arg)))
print('------------')
with open('models/%s/args.txt' % args.name, 'w') as f:
for arg in vars(args):
print('- %s: %s' % (arg, getattr(args, arg)), file=f)
joblib.dump(args, 'models/%s/args.pkl' % args.name)
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpus
# switch to benchmark model, a little forward results fluctuation, a little fast training
cudnn.benchmark = True
# switch to deterministic model, more stable
# cudnn.deterministic = True
img_path, img_labels, num_outputs = img_path_generator(
dataset=args.train_dataset)
if args.pred_type == 'regression':
num_outputs = 1
skf = StratifiedKFold(n_splits=args.n_splits,
shuffle=True,
random_state=30)
img_paths = []
labels = []
for fold, (train_idx,
val_idx) in enumerate(skf.split(img_path, img_labels)):
img_paths.append((img_path[train_idx], img_path[val_idx]))
labels.append((img_labels[train_idx], img_labels[val_idx]))
train_transform = []
train_transform = transforms.Compose([
transforms.Resize((args.img_size, args.img_size)),
# transforms.RandomAffine(
# degrees=(args.rotate_min, args.rotate_max) if args.rotate else 0,
# translate=(args.translate_min, args.translate_max) if args.translate else None,
# scale=(args.rescale_min, args.rescale_max) if args.rescale else None,
# shear=(args.shear_min, args.shear_max) if args.shear else None,
# ),
transforms.RandomCrop(args.input_size, padding=4),
transforms.RandomHorizontalFlip(),
# transforms.RandomVerticalFlip(),
# transforms.ColorJitter(
# brightness=0,
# contrast=args.contrast,
# saturation=0,
# hue=0),
RandomErase(prob=args.random_erase_prob if args.random_erase else 0,
sl=args.random_erase_sl,
sh=args.random_erase_sh,
r=args.random_erase_r),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
])
val_transform = transforms.Compose([
transforms.Resize((args.img_size, args.img_size)),
transforms.CenterCrop(args.input_size),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
])
if args.loss == 'CrossEntropyLoss':
criterion = nn.CrossEntropyLoss().cuda()
elif args.loss == 'FocalLoss':
criterion = FocalLoss().cuda()
elif args.loss == 'MSELoss':
criterion = nn.MSELoss().cuda()
elif args.loss == 'LabelSmoothingLoss':
criterion = LabelSmoothingLoss(classes=num_outputs,
smoothing=0.8).cuda()
else:
raise NotImplementedError
folds = []
best_losses = []
best_ac_scores = []
best_epochs = []
for fold, ((train_img_paths, val_img_paths),
(train_labels, val_labels)) in enumerate(zip(img_paths,
labels)):
print('Fold [%d/%d]' % (fold + 1, len(img_paths)))
# if os.path.exists('models/%s/model_%d.pth' % (args.name, fold+1)):
# log = pd.read_csv('models/%s/log_%d.csv' % (args.name, fold+1))
# best_loss, best_ac_score = log.loc[log['val_loss'].values.argmin(
# ), ['val_loss', 'val_score', 'val_ac_score']].values
# folds.append(str(fold + 1))
# best_losses.append(best_loss)
# best_ac_scores.append(best_ac_score)
# continue
# train
train_set = Dataset(train_img_paths,
train_labels,
transform=train_transform)
train_loader = torch.utils.data.DataLoader(train_set,
batch_size=args.batch_size,
shuffle=True,
num_workers=0,
sampler=None)
val_set = Dataset(val_img_paths, val_labels, transform=val_transform)
val_loader = torch.utils.data.DataLoader(val_set,
batch_size=args.batch_size,
shuffle=False,
num_workers=0)
# create model
if args.mode == 'baseline':
model = get_model(model_name=args.arch,
num_outputs=num_outputs,
freeze_bn=args.freeze_bn,
dropout_p=args.dropout_p)
elif args.mode == 'gcn':
model_path = 'models/%s/model_%d.pth' % ('baseline_' + args.arch,
fold + 1)
if not os.path.exists(model_path):
print('%s is not exists' % model_path)
continue
model = SoftLabelGCN(cnn_model_name=args.arch,
cnn_pretrained=False,
num_outputs=num_outputs)
pretrained_dict = torch.load(model_path)
model_dict = model.cnn.state_dict()
pretrained_dict = {
k: v
for k, v in pretrained_dict.items() if k in model_dict
}
model_dict.update(pretrained_dict)
model.cnn.load_state_dict(model_dict)
for p in model.cnn.parameters():
p.requires_grad = False
else:
# model = RA(cnn_model_name=args.arch, input_size=args.input_size, hidden_size=args.lstm_hidden,
# layer_num=args.lstm_layers, recurrent_num=args.lstm_recurrence, class_num=num_outputs, pretrain=True)
model_path = 'models/%s/model_%d.pth' % ('baseline_' + args.arch,
fold + 1)
if not os.path.exists(model_path):
print('%s is not exists' % model_path)
continue
model = RA(cnn_model_name=args.arch,
input_size=args.input_size,
hidden_size=args.lstm_hidden,
layer_num=args.lstm_layers,
recurrent_num=args.lstm_recurrence,
class_num=num_outputs)
pretrained_dict = torch.load(model_path)
model_dict = model.cnn.state_dict()
pretrained_dict = {
k: v
for k, v in pretrained_dict.items() if k in model_dict
}
model_dict.update(pretrained_dict)
model.cnn.load_state_dict(model_dict)
for p in model.cnn.parameters():
p.requires_grad = False
device = torch.device('cuda')
if torch.cuda.device_count() > 1:
model = nn.DataParallel(model)
model.to(device)
# model = model.cuda()
if args.pretrained_model is not None:
model.load_state_dict(
torch.load('models/%s/model_%d.pth' %
(args.pretrained_model, fold + 1)))
# print(model)
if args.optimizer == 'Adam':
optimizer = optim.Adam(filter(lambda p: p.requires_grad,
model.parameters()),
lr=args.lr)
elif args.optimizer == 'AdamW':
optimizer = optim.AdamW(filter(lambda p: p.requires_grad,
model.parameters()),
lr=args.lr)
elif args.optimizer == 'RAdam':
optimizer = RAdam(filter(lambda p: p.requires_grad,
model.parameters()),
lr=args.lr)
elif args.optimizer == 'SGD':
optimizer = optim.SGD(filter(lambda p: p.requires_grad,
model.parameters()),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay,
nesterov=args.nesterov)
# optimizer = optim.SGD(model.get_config_optim(args.lr, args.lr * 10, args.lr * 10), lr=args.lr,
# momentum=args.momentum, weight_decay=args.weight_decay, nesterov=args.nesterov)
if args.scheduler == 'CosineAnnealingLR':
scheduler = lr_scheduler.CosineAnnealingLR(optimizer,
T_max=args.epochs,
eta_min=args.min_lr)
elif args.scheduler == 'ReduceLROnPlateau':
scheduler = lr_scheduler.ReduceLROnPlateau(optimizer,
factor=args.factor,
patience=args.patience,
verbose=1,
min_lr=args.min_lr)
log = pd.DataFrame(index=[],
columns=[
'epoch',
'loss',
'ac_score',
'val_loss',
'val_ac_score',
])
log = {
'epoch': [],
'loss': [],
'ac_score': [],
'val_loss': [],
'val_ac_score': [],
}
best_loss = float('inf')
best_ac_score = 0
best_epoch = 0
for epoch in range(args.epochs):
print('Epoch [%d/%d]' % (epoch + 1, args.epochs))
# train for one epoch
train_loss, train_ac_score = train(args, train_loader, model,
criterion, optimizer, epoch)
# evaluate on validation set
val_loss, val_ac_score = validate(args, val_loader, model,
criterion)
if args.scheduler == 'CosineAnnealingLR':
scheduler.step()
elif args.scheduler == 'ReduceLROnPlateau':
scheduler.step(val_loss)
print(
'loss %.4f - ac_score %.4f - val_loss %.4f - val_ac_score %.4f'
% (train_loss, train_ac_score, val_loss, val_ac_score))
log['epoch'].append(epoch)
log['loss'].append(train_loss)
log['ac_score'].append(train_ac_score)
log['val_loss'].append(val_loss)
log['val_ac_score'].append(val_ac_score)
pd.DataFrame(log).to_csv('models/%s/log_%d.csv' %
(args.name, fold + 1),
index=False)
if val_ac_score > best_ac_score:
if args.mode == 'baseline':
torch.save(
model.state_dict(),
'models/%s/model_%d.pth' % (args.name, fold + 1))
best_loss = val_loss
best_ac_score = val_ac_score
best_epoch = epoch
print("=> saved best model")
print('val_loss: %f' % best_loss)
print('val_ac_score: %f' % best_ac_score)
folds.append(str(fold + 1))
best_losses.append(best_loss)
best_ac_scores.append(best_ac_score)
best_epochs.append(best_epoch)
results = pd.DataFrame({
'fold':
folds + ['mean'],
'best_loss':
best_losses + [np.mean(best_losses)],
'best_ac_score':
best_ac_scores + [np.mean(best_ac_scores)],
'best_epoch':
best_epochs + [''],
})
print(results)
results.to_csv('models/%s/results.csv' % args.name, index=False)
torch.cuda.empty_cache()
if not args.cv:
break
class TradingBot:
def __init__(self, default_params, specific_params, db_client, exchange_client=None, is_backtest=False):
# initialize status and settings of bot.
# if you try backtest, db_client is in need.
self.exchange_client = exchange_client
self.db_client = db_client
self.default_params = default_params
self.extract_default_params(self.default_params)
self.specific_params = specific_params
self.combined_params = dict(**self.default_params, **self.specific_params)
self.is_backtest = is_backtest
if is_backtest:
# for params table
self.backtest_management_table_name = self.bot_name + "_backtest_management"
# backtest configure
self.initial_balance = 100.0 # USD
self.account_currency = "USD"
self.dataset_manipulator = Dataset(self.db_client, self.exchange_client)
if self.db_client.is_table_exist(self.backtest_management_table_name) is not True:
self.create_backtest_management_table()
if self.db_client.is_table_exist("backtest_management") is True:
# delete useless template table
drop_query = "DROP TABLE backtest_management;"
self.db_client.exec_sql(drop_query, return_df=False)
def create_backtest_management_table(self):
backtest_management_template = BacktestManagement
table_def = backtest_management_template.__table__
# add specific params columns
table_def = self.append_specific_params_column(table_def)
backtest_summary_id = Column("backtest_summary_id", Integer)
table_def.relation = relationship("BacktestSummary")
table_def.append_column(backtest_summary_id)
table_def.name = self.backtest_management_table_name
table_def.create(bind=self.db_client.connector)
# add foreign key constraint
ctx = MigrationContext.configure(self.db_client.connector)
op = Operations(ctx)
with op.batch_alter_table(self.bot_name + "_backtest_management") as batch_op:
batch_op.create_foreign_key("fk_management_summary", "backtest_summary", ["backtest_summary_id"], ["id"])
def backtest_management_table(self):
return Table(self.backtest_management_table_name, MetaData(bind=self.db_client.connector),
autoload=True, autoload_with=self.db_client.connector)
def append_specific_params_column(self, table_def):
return table_def
# Need to be oberride
# return table def
def extract_default_params(self, default_params):
# default_params = {
# "bot_name" : bot_name, # used in bot name builder for log
# "timeframe": integer,
# "close_position_on_do_nothing": close_position_on_do_nothing,
# "inverse_trading": inverse_trading
# }
self.bot_name = default_params["bot_name"]
self.timeframe = default_params["timeframe"]
self.close_position_on_do_nothing = default_params["close_position_on_do_nothing"]
self.inverse_trading = default_params["inverse_trading"]
def run(self, ohlcv_df=None, backtest_start_time=datetime.now() - timedelta(days=90), backtest_end_time=datetime.now(),
floor_time=True):
if ohlcv_df is not None:
self.ohlcv_df = ohlcv_df
else:
self.ohlcv_df = self.dataset_manipulator.get_ohlcv(self.timeframe, backtest_start_time, backtest_end_time)
self.ohlcv_with_metrics = self.calculate_metrics_for_backtest()
if self.is_backtest:
# for summary
if floor_time:
backtest_start_time = self.dataset_manipulator.floor_datetime_to_ohlcv(backtest_start_time, "up")
backtest_end_time = self.dataset_manipulator.floor_datetime_to_ohlcv(backtest_end_time, "down")
self.backtest_start_time = backtest_start_time
self.backtest_end_time = backtest_end_time
self.ohlcv_with_signals = self.calculate_signs_for_backtest().dropna()
self.summary_id = self.init_summary()
self.insert_backtest_transaction_logs()
self.insert_params_management()
self.update_summary()
def bulk_insert(self):
self.db_client.session.commit()
def reset_backtest_result_with_params(self, default_params, specific_params):
# for loop and serach optimal metrics value
self.ohlcv_with_metrics = None
self.ohlcv_with_signals = None
self.summary_id = None
self.closed_positions_df = None
self.default_params = default_params
self.extract_default_params(self.default_params)
self.specific_params = specific_params
self.combined_params = dict(**self.default_params, **self.specific_params)
def insert_params_management(self):
backtest_management = self.backtest_management_table()
self.combined_params["backtest_summary_id"] = int(self.summary_id)
del self.combined_params["bot_name"]
self.db_client.connector.execute(backtest_management.insert().values(self.combined_params))
def init_summary(self):
summary = BacktestSummary().__table__
init_summary = {
"bot_name": self.bot_name,
"initial_balance": self.initial_balance,
"account_currency": self.account_currency
}
self.db_client.connector.execute(summary.insert().values(init_summary))
# [FIXME] only for single task processing, unable to parallel process
return int(self.db_client.get_last_row("backtest_summary").index.array[0])
def calculate_lot(self):
return 1 # 100 %
# if you need, you can override
# default is invest all that you have
def calculate_leverage(self):
return 1 # 1 times
# if you need, you can override
def calculate_metric(self):
return "metric"
def calculate_metrics_for_backtest(self):
return "ohlcv_with_metric_dataframe"
# need to override
def calculate_sign(self):
return "signal"
# need to override
# return ["buy", "sell", "do_nothing"]
def calculate_signs_for_backtest(self):
return "ohlcv_with_signal_data"
# need to override
# return dataframe with ["buy", "sell", "do_nothing"]
def insert_backtest_transaction_logs(self):
# refer to signal then judge investment
# keep one order at most
position = None
current_balance = self.initial_balance
transaction_logs = []
for row in self.ohlcv_with_signals.itertuples(): # self.ohlcv_with_signals should be dataframe
if row.signal == "buy":
if position is not None and position.order_type == "long":
# inverse => close position
if self.inverse_trading:
position.close_position(row)
transaction_logs.append(position.generate_transaction_log(self.db_client, self.summary_id))
current_balance = position.current_balance
position = None
# normal => still holding
else:
pass
elif position is not None and position.order_type == "short":
# inverse => still holding
if self.inverse_trading:
pass
# normal => close position
else:
position.close_position(row)
transaction_logs.append(position.generate_transaction_log(self.db_client, self.summary_id))
current_balance = position.current_balance
position = None
else:
lot = self.calculate_lot()
leverage = self.calculate_leverage()
# inverse => open short position
if self.inverse_trading:
position = OrderPosition(row, "short", current_balance, lot, leverage, is_backtest=True)
else:
# normal => open long position
position = OrderPosition(row, "long", current_balance, lot, leverage, is_backtest=True)
elif row.signal == "sell":
if position is not None and position.order_type == "long":
# inverse => still holding
if self.inverse_trading:
pass
# normal => close position
else:
position.close_position(row)
transaction_logs.append(position.generate_transaction_log(self.db_client, self.summary_id))
current_balance = position.current_balance
position = None
elif position is not None and position.order_type == "short":
# inverse => close position
if self.inverse_trading:
position.close_position(row)
transaction_logs.append(position.generate_transaction_log(self.db_client, self.summary_id))
current_balance = position.current_balance
position = None
# normal => still holding
else:
pass
else:
lot = self.calculate_lot()
leverage = self.calculate_leverage()
# inverse => open long position
if self.inverse_trading:
position = OrderPosition(row, "long",current_balance, lot, leverage, is_backtest=True)
else:
# normal => open short position
position = OrderPosition(row, "short",current_balance, lot, leverage, is_backtest=True)
elif row.signal == "do_nothing":
if self.close_position_on_do_nothing:
# if do nothing option is true
# and you get do nothing from signal, then close out the position
if position is not None:
# close position
position.close_position(row)
transaction_logs.append(position.generate_transaction_log(self.db_client, self.summary_id))
current_balance = position.current_balance
position = None
# the processing time is propotionate to the number of transaction logs
self.db_client.session.bulk_insert_mappings(BacktestTransactionLog, transaction_logs)
self.closed_positions_df = pd.DataFrame(transaction_logs)
self.closed_positions_df["holding_time"] = self.closed_positions_df["close_time"] - \
self.closed_positions_df["entry_time"]
def update_summary(self):
win_entries_condition = (self.closed_positions_df["profit_status"] == "win")
win_row = self.closed_positions_df[(win_entries_condition)]
lose_entries_condition = (self.closed_positions_df["profit_status"] == "lose")
lose_row = self.closed_positions_df[(lose_entries_condition)]
long_entries_condition = (self.closed_positions_df["order_type"] == "long")
long_row = self.closed_positions_df[(long_entries_condition)]
short_entries_condition = (self.closed_positions_df["order_type"] == "short")
short_row = self.closed_positions_df[(short_entries_condition)]
win_long_row = self.closed_positions_df[(win_entries_condition) & (long_entries_condition)]
win_short_row = self.closed_positions_df[(win_entries_condition) & (short_entries_condition)]
lose_long_row = self.closed_positions_df[(lose_entries_condition) & (long_entries_condition)]
lose_short_row = self.closed_positions_df[(lose_entries_condition) & (short_entries_condition)]
total_return = float(self.closed_positions_df.profit_size.sum())
if self.closed_positions_df.iloc[-1].current_balance > 0:
total_return_percentage = float(100 * (self.closed_positions_df.iloc[-1].current_balance / self.initial_balance))
else:
total_return_percentage = \
float(-100 * (abs(self.closed_positions_df.iloc[-1].current_balance) + self.initial_balance) / self.initial_balance)
win_consecutive = self.build_consecutive("win")
lose_consecutive = self.build_consecutive("lose")
drawdowns = self.build_drawdowns()
if drawdowns["maximal_drawdown"] == 0:
recovery_factor = 0
else:
recovery_factor = total_return / drawdowns["maximal_drawdown"]
# total
summary_dict = {
"id": self.summary_id,
"total_entry": len(self.closed_positions_df),
"total_max_holding_ms": self.closed_positions_df["holding_time"].max().to_pytimedelta(),
"total_average_holding_ms": self.closed_positions_df["holding_time"].mean().to_pytimedelta(),
"total_min_holding_ms": self.closed_positions_df["holding_time"].min().to_pytimedelta(),
"total_return": total_return,
"total_return_average": float(self.closed_positions_df.profit_size.mean()),
"total_standard_deviation": float(self.closed_positions_df.profit_size.std()),
"total_skewness": float(self.closed_positions_df.profit_size.skew()),
"total_kurtosis": float(self.closed_positions_df.profit_size.kurt()),
"total_median": float(self.closed_positions_df.profit_size.median()),
"total_return_percentage": total_return_percentage,
"total_return_average_percentage": float(self.closed_positions_df.profit_percentage.mean()),
"total_standard_deviation_percentage": float(self.closed_positions_df.profit_percentage.std()),
"total_skewness_percentage": float(self.closed_positions_df.profit_percentage.skew()),
"total_kurtosis_percentage": float(self.closed_positions_df.profit_percentage.kurt()),
"total_median_percentage": float(self.closed_positions_df.profit_percentage.median()),
"total_transaction_cost": float(self.closed_positions_df.transaction_cost.sum()),
# win
"win_entry": len(win_row),
"win_average_holding_ms": win_row["holding_time"].mean().to_pytimedelta(),
"win_rate": (len(win_row) / len(self.closed_positions_df)) * 100,
"win_return": float(win_row.profit_size.sum()),
"win_return_average": float(win_row.profit_size.mean()),
"win_standard_deviation": float(win_row.profit_size.std()),
"win_skewness": float(win_row.profit_size.skew()),
"win_kurtosis": float(win_row.profit_size.kurt()),
"win_median": float(win_row.profit_size.median()),
"win_return_average_percentage": float(win_row.profit_percentage.mean()),
"win_standard_deviation_percentage": float(win_row.profit_percentage.std()),
"win_skewness_percentage": float(win_row.profit_percentage.skew()),
"win_kurtosis_percentage": float(win_row.profit_percentage.kurt()),
"win_median_percentage": float(win_row.profit_percentage.median()),
"win_transaction_cost": float(win_row.transaction_cost.sum()),
"win_consecutive_max_entry": win_consecutive["consecutive_max_entry"],
"win_consecutive_average_entry": win_consecutive["consecutive_average_entry"],
"win_consecutive_max_profit": float(win_consecutive["consecutive_df"].profit_size.sum()),
"win_max_profit": float(win_row.profit_size.max()),
"win_max_profit_percentage": float(win_row.profit_percentage.max()),
# lose
"lose_entry": len(lose_row),
"lose_rate": (len(lose_row) / len(self.closed_positions_df)) * 100,
"lose_average_holding_ms": lose_row["holding_time"].mean().to_pytimedelta(),
"lose_return": float(lose_row.profit_size.sum()),
"lose_return_average": float(lose_row.profit_size.mean()),
"lose_standard_deviation": float(lose_row.profit_size.std()),
"lose_skewness": float(lose_row.profit_size.skew()),
"lose_kurtosis": float(lose_row.profit_size.kurt()),
"lose_median": float(lose_row.profit_size.median()),
"lose_return_average_percentage": float(lose_row.profit_percentage.mean()),
"lose_standard_deviation_percentage": float(lose_row.profit_percentage.std()),
"lose_skewness_percentage": float(lose_row.profit_percentage.skew()),
"lose_kurtosis_percentage": float(lose_row.profit_percentage.kurt()),
"lose_median_percentage": float(lose_row.profit_percentage.median()),
"lose_transaction_cost": float(lose_row.transaction_cost.sum()),
"lose_consecutive_max_entry": lose_consecutive["consecutive_max_entry"],
"lose_consecutive_average_entry": lose_consecutive["consecutive_average_entry"],
"lose_consecutive_max_loss": float(lose_consecutive["consecutive_df"].profit_size.sum()),
"lose_max_loss": float(lose_row.profit_size.min()),
"lose_max_loss_percentage": float(lose_row.profit_percentage.min()),
# long
"long_entry": len(long_row),
"long_rate": (len(long_row) / len(self.closed_positions_df)) * 100,
"long_average_holding_ms": long_row["holding_time"].mean().to_pytimedelta(),
"long_return": float(long_row.profit_size.sum()),
"long_return_average": float(long_row.profit_size.mean()),
"long_standard_deviation": float(long_row.profit_size.std()),
"long_skewness": float(long_row.profit_size.skew()),
"long_kurtosis": float(long_row.profit_size.kurt()),
"long_median": float(long_row.profit_size.median()),
"long_return_average_percentage": float(long_row.profit_percentage.mean()),
"long_standard_deviation_percentage": float(long_row.profit_percentage.std()),
"long_skewness_percentage": float(long_row.profit_percentage.skew()),
"long_kurtosis_percentage": float(long_row.profit_percentage.kurt()),
"long_median_percentage": float(long_row.profit_percentage.median()),
"long_max_profit": float(long_row.profit_size.max()),
"long_max_profit_percentage": float(long_row.profit_percentage.max()),
"long_max_loss": float(long_row.profit_size.min()),
"long_max_loss_percentage": float(long_row.profit_percentage.min()),
#short
"short_entry": len(short_row),
"short_rate": (len(short_row) / len(self.closed_positions_df)) * 100,
"short_average_holding_ms": short_row["holding_time"].mean().to_pytimedelta(),
"short_return": float(short_row.profit_size.sum()),
"short_return_average": float(short_row.profit_size.mean()),
"short_standard_deviation": float(short_row.profit_size.std()),
"short_skewness": float(short_row.profit_size.skew()),
"short_kurtosis": float(short_row.profit_size.kurt()),
"short_median": float(short_row.profit_size.median()),
"short_return_average_percentage": float(short_row.profit_percentage.mean()),
"short_standard_deviation_percentage": float(short_row.profit_percentage.std()),
"short_skewness_percentage": float(short_row.profit_percentage.skew()),
"short_kurtosis_percentage": float(short_row.profit_percentage.kurt()),
"short_median_percentage": float(short_row.profit_percentage.median()),
"short_max_profit": float(short_row.profit_size.max()),
"short_max_profit_percentage": float(short_row.profit_percentage.max()),
"short_max_loss": float(short_row.profit_size.min()),
"short_max_loss_percentage": float(short_row.profit_percentage.min()),
# win long
"win_long_entry": len(win_long_row),
"win_long_average_holding_ms": win_long_row["holding_time"].mean().to_pytimedelta(),
"win_long_return": float(win_long_row.profit_size.sum()),
"win_long_return_average": float(win_long_row.profit_size.mean()),
"win_long_standard_deviation": float(win_long_row.profit_size.std()),
"win_long_skewness": float(win_long_row.profit_size.skew()),
"win_long_kurtosis": float(win_long_row.profit_size.kurt()),
"win_long_median": float(win_long_row.profit_size.median()),
"win_long_return_average": float(win_long_row.profit_size.mean()),
"win_long_return_average_percentage": float(win_long_row.profit_percentage.mean()),
"win_long_standard_deviation_percentage": float(win_long_row.profit_percentage.std()),
"win_long_skewness_percentage": float(win_long_row.profit_percentage.skew()),
"win_long_kurtosis_percentage": float(win_long_row.profit_percentage.kurt()),
"win_long_median_percentage": float(win_long_row.profit_percentage.median()),
# win short
"win_short_entry": len(win_short_row),
"win_short_average_holding_ms": win_short_row["holding_time"].mean().to_pytimedelta(),
"win_short_return": float(win_short_row.profit_size.sum()),
"win_short_return_average": float(win_short_row.profit_size.mean()),
"win_short_standard_deviation": float(win_short_row.profit_size.std()),
"win_short_skewness": float(win_short_row.profit_size.skew()),
"win_short_kurtosis": float(win_short_row.profit_size.kurt()),
"win_short_median": float(win_short_row.profit_size.median()),
"win_short_return_average_percentage": float(win_short_row.profit_percentage.mean()),
"win_short_standard_deviation_percentage": float(win_short_row.profit_percentage.std()),
"win_short_skewness_percentage": float(win_short_row.profit_percentage.skew()),
"win_short_kurtosis_percentage": float(win_short_row.profit_percentage.kurt()),
"win_short_median_percentage": float(win_short_row.profit_percentage.median()),
# lose long
"lose_long_entry": len(lose_long_row),
"lose_long_average_holding_ms": lose_long_row["holding_time"].mean().to_pytimedelta(),
"lose_long_return": float(lose_long_row.profit_size.sum()),
"lose_long_return_average": float(lose_long_row.profit_size.mean()),
"lose_long_standard_deviation": float(lose_long_row.profit_size.std()),
"lose_long_skewness": float(lose_long_row.profit_size.skew()),
"lose_long_kurtosis": float(lose_long_row.profit_size.kurt()),
"lose_long_median": float(lose_long_row.profit_size.median()),
"lose_long_return_average_percentage": float(lose_long_row.profit_percentage.mean()),
"lose_long_standard_deviation_percentage": float(lose_long_row.profit_percentage.std()),
"lose_long_skewness_percentage": float(lose_long_row.profit_percentage.skew()),
"lose_long_kurtosis_percentage": float(lose_long_row.profit_percentage.kurt()),
"lose_long_median_percentage": float(lose_long_row.profit_percentage.median()),
# lose short
"lose_short_entry": len(lose_short_row),
"lose_short_average_holding_ms": lose_short_row["holding_time"].mean().to_pytimedelta(),
"lose_short_return": float(lose_short_row.profit_size.sum()),
"lose_short_return_average": float(lose_short_row.profit_size.mean()),
"lose_short_standard_deviation": float(lose_short_row.profit_size.std()),
"lose_short_skewness": float(lose_short_row.profit_size.skew()),
"lose_short_kurtosis": float(lose_short_row.profit_size.kurt()),
"lose_short_median": float(lose_short_row.profit_size.median()),
"lose_short_return_average_percentage": float(lose_short_row.profit_percentage.mean()),
"lose_short_standard_deviation_percentage": float(lose_short_row.profit_percentage.std()),
"lose_short_skewness_percentage": float(lose_short_row.profit_percentage.skew()),
"lose_short_kurtosis_percentage": float(lose_short_row.profit_percentage.kurt()),
"lose_short_median_percentage": float(lose_short_row.profit_percentage.median()),
# other metrics
"backtest_start_time": self.backtest_start_time,
"backtest_end_time": self.backtest_end_time,
"bot_name": self.bot_name,
"initial_balance": self.initial_balance,
"account_currency": self.account_currency,
"absolute_drawdown": float(drawdowns["absolute_drawdown"]),
"maximal_drawdown": float(drawdowns["maximal_drawdown"]),
"relative_drawdown": float(drawdowns["relative_drawdown"]),
"profit_factor": float(win_row.profit_size.sum() / abs(lose_row.profit_size.sum())),
"recovery_factor": recovery_factor
}
self.db_client.session.bulk_update_mappings(BacktestSummary, [summary_dict])
def build_drawdowns(self):
if self.closed_positions_df.current_balance.min() < 0:
absolute_drawdown = self.initial_balance + self.closed_positions_df.current_balance.min()
else:
absolute_drawdown = self.initial_balance - self.closed_positions_df.current_balance.min()
current_drawdown = 0
current_relative_drawdown = 0
max_balance = self.initial_balance
maximal_drawdown = 0
relative_drawdown = 0
for log in self.closed_positions_df.itertuples():
if max_balance < log.current_balance:
max_balance = log.current_balance
else:
if log.current_balance > 0:
current_drawdown = max_balance - log.current_balance
else:
current_drawdown = max_balance + log.current_balance
current_relative_drawdown = (abs(current_drawdown) / max_balance)*100
if maximal_drawdown < current_drawdown:
maximal_drawdown = current_drawdown
if relative_drawdown < current_relative_drawdown:
relative_drawdown = current_relative_drawdown
result = {
"absolute_drawdown" : absolute_drawdown,
"maximal_drawdown": maximal_drawdown,
"relative_drawdown" : relative_drawdown
}
return result
def build_consecutive(self, profit_status):
current_start_index = 0
current_end_index = 0
max_start_index = 0
max_consecutive = 0
max_end_index = 0
consective_flag = False
consecutive_win_lose_entries = []
profit_status_df = self.closed_positions_df.loc[:,["profit_status"]]
profit_status_np = profit_status_df.to_numpy(copy=True)
# for loop
for row_id, row in enumerate(profit_status_np):
if row[0] == profit_status:
if consective_flag is False:
# consecutive count start
current_start_index = row_id
consective_flag = True
else:
if consective_flag:
current_end_index = row_id
if max_consecutive <= current_end_index - current_start_index:
max_start_index = current_start_index
max_end_index = current_end_index - 1
max_consecutive = current_end_index - current_start_index
consecutive_win_lose_entries.append(current_end_index - current_start_index)
consective_flag = False
consecutive_max_entry = np.max(consecutive_win_lose_entries)
consecutive_average_entry = np.mean(consecutive_win_lose_entries)
return_hash = {
"consecutive_df": self.closed_positions_df.loc[max_start_index:max_end_index],
"consecutive_max_entry": int(consecutive_max_entry),
"consecutive_average_entry": float(consecutive_average_entry)
}
return return_hash
def main():
parser = argparse.ArgumentParser(
description=
"Generate some useful quantile graphs and perhaps a few histograms too"
)
parser.add_argument("data_directory",
help="the location where we are storing the data.")
parser.add_argument(
"output_directory",
help="the location where we will want to output the graphs.")
args = parser.parse_args()
def make_graph(az, insttype, data_before, data_after, interval, axes):
print("building a graph for az: %s insttype: %s" % (az, insttype))
print(
"\twe have %d datapoints from before, and %d datapoints from after"
% (len(data_before), len(data_after)))
# create the price data models
data_before = PriceTimeSeriesNaive(data_before)
data_after = PriceTimeSeriesNaive(data_after)
expanded_before = list(data_before.expand_with_interval(350))
expanded_after = list(data_after.expand_with_interval(350))
print("\tbefore expanded length: %d" % len(expanded_before))
print("\tafter expanded length: %d" % len(expanded_after))
buckets_before = [
PriceBucketNaive(list(bucket))
for idx, bucket in itertools.groupby(
expanded_before, lambda x: x["timestamp"] // interval)
]
buckets_after = [
PriceBucketNaive(list(bucket))
for idx, bucket in itertools.groupby(
expanded_after, lambda x: x["timestamp"] // interval)
]
expanded_before, expanded_after = None, None # we null them so that their memory can be collected if need be
print("\tbuckets before length: %d" % len(buckets_before))
print("\tbuckets after length: %d" % len(buckets_after))
min_len = min(len(buckets_before), len(buckets_after))
buckets_before = buckets_before[-min_len:]
buckets_after = buckets_after[-min_len:]
series_avg_before = [bucket.getAverage() for bucket in buckets_before]
series_avg_after = [bucket.getAverage() for bucket in buckets_after]
series_p50_before = [
bucket.getQuantile(0.5)["price"] for bucket in buckets_before
]
series_p50_after = [
bucket.getQuantile(0.5)["price"] for bucket in buckets_after
]
series_p66_before = [
bucket.getQuantile(0.66)["price"] for bucket in buckets_before
]
series_p66_after = [
bucket.getQuantile(0.66)["price"] for bucket in buckets_after
]
series_p95_before = [
bucket.getQuantile(0.95)["price"] for bucket in buckets_before
]
series_p95_after = [
bucket.getQuantile(0.95)["price"] for bucket in buckets_after
]
# f = plt.figure(figsize=(5, 10))
xvalues = list(range(-min_len, 0))
axes[0].set_title("%s-%s averages\nduration: %d" %
(az, insttype, interval / 3600))
axes[0].set_ylabel("Price $")
axes[0].plot(xvalues, series_avg_before, label="before")
axes[0].plot(xvalues, series_avg_after, label="after")
axes[1].set_title("%s-%s q.5\nduration: %d" %
(az, insttype, interval / 3600))
axes[1].set_ylabel("Price $")
axes[1].plot(xvalues, series_p50_before, label="before")
axes[1].plot(xvalues, series_p50_after, label="after")
axes[2].set_title("%s-%s q.66\nduration: %d" %
(az, insttype, interval / 3600))
axes[2].set_ylabel("Price $")
axes[2].plot(xvalues, series_p66_before, label="before")
axes[2].plot(xvalues, series_p66_after, label="after")
axes[3].set_title("%s-%s q.95\nduration: %d" %
(az, insttype, interval / 3600))
axes[3].set_ylabel("Price $")
axes[3].plot(xvalues, series_p95_before, label="before")
axes[3].plot(xvalues, series_p95_after, label="after")
# plt.legend()
# plt.tight_layout()
# f.savefig(path.join(args.output_directory, "%s_%s_interval%d.png" % (az, insttype, interval / 3600)), bbox_inches="tight", dpi=150)
# plt.close()
def transform_query_results(results):
return ({
"timestamp": result[0],
"price": result[1]
} for result in results)
with Dataset(args.data_directory) as dataset:
for az, insttype in dataset.get_databases(
): # misleading function name
conn = dataset.open(az, insttype)
# compute the most recent timestamp so we can determine a good
# duration for the intervals we will be examining
c = conn.cursor()
c.execute("SELECT MAX(timestamp) AS timestamp FROM prices")
most_recent_ts = c.fetchone()[0]
interval_duration = most_recent_ts - AFTER_START_EPOCH
if interval_duration < 0: continue
# fetch the data for the before window
c = conn.cursor()
c.execute(
"SELECT timestamp, price FROM prices "
" WHERE timestamp > %d AND timestamp < %d ORDER BY timestamp" %
(BEFORE_END_EPOCH - interval_duration, BEFORE_END_EPOCH))
data_before = list(transform_query_results(c.fetchall()))
c = conn.cursor()
c.execute("SELECT timestamp, price FROM prices " +
" WHERE timestamp > %d ORDER BY timestamp " %
(AFTER_START_EPOCH))
data_after = list(transform_query_results(c.fetchall()))
if len(data_before) == 0 or len(data_after) == 0: continue
print("producing a graph for %s - %s" % (az, insttype))
f, axes = plt.subplots(4, 4, figsize=(24, 12))
axes = axes.flatten()
make_graph(az, insttype, list(data_before), list(data_after), 3600,
axes[0::4])
make_graph(az, insttype, list(data_before), list(data_after),
8 * 3600, axes[1::4])
make_graph(az, insttype, list(data_before), list(data_after),
24 * 3600, axes[2::4])
make_graph(az, insttype, list(data_before), list(data_after),
3 * 24 * 3600, axes[3::4])
plt.legend()
plt.tight_layout()
f.savefig(path.join(args.output_directory,
"%s_%s.png" % (az, insttype)),
bbox_inches="tight",
dpi=150)
plt.close()
def main():
test_args = parse_args()
args = joblib.load('models/%s/args.pkl' % test_args.name)
print('Config -----')
for arg in vars(args):
print('%s: %s' % (arg, getattr(args, arg)))
print('------------')
if args.pred_type == 'all':
num_outputs = 6
elif args.pred_type == 'except_any':
num_outputs = 5
else:
raise NotImplementedError
# create model
model_path = 'models/%s/model.pth' % args.name
model = get_model(model_name=args.arch,
num_outputs=num_outputs,
freeze_bn=args.freeze_bn,
dropout_p=args.dropout_p,
pooling=args.pooling,
lp_p=args.lp_p)
model = model.cuda()
model.load_state_dict(torch.load(model_path))
model.eval()
cudnn.benchmark = True
test_transform = Compose([
transforms.Resize(args.img_size, args.img_size),
ForegroundCenterCrop(args.crop_size),
transforms.Normalize(mean=model.mean, std=model.std),
ToTensor(),
])
# data loading code
# if args.img_type:
# stage_1_test_dir = 'processed/stage_1_test_%s' %args.img_type
# else:
# stage_1_test_dir = 'processed/stage_1_test'
if args.img_type:
stage_2_test_dir = 'processed/stage_2_test_%s' % args.img_type
else:
stage_2_test_dir = 'processed/stage_2_test'
# test_df = pd.read_csv('inputs/stage_1_sample_submission.csv')
test_df = pd.read_csv('inputs/stage_2_sample_submission.csv')
# test_img_paths = np.array([stage_1_test_dir + '/' + '_'.join(s.split('_')[:-1]) + '.png' for s in test_df['ID']][::6])
test_img_paths = np.array([
stage_2_test_dir + '/' + '_'.join(s.split('_')[:-1]) + '.png'
for s in test_df['ID']
][::6])
test_labels = np.array([
test_df.loc[c::6, 'Label'].values for c in range(6)
]).T.astype('float32')
test_set = Dataset(test_img_paths, test_labels, transform=test_transform)
test_loader = torch.utils.data.DataLoader(test_set,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers)
preds = []
preds_fold = []
with torch.no_grad():
for i, (input, _) in tqdm(enumerate(test_loader),
total=len(test_loader)):
outputs = []
for input in apply_tta(args, input):
input = input.cuda()
output = model(input)
output = torch.sigmoid(output)
if args.pred_type == 'except_any':
output = torch.cat(
[output, torch.max(output, 1, keepdim=True)[0]], 1)
outputs.append(output.data.cpu().numpy())
preds_fold.extend(np.mean(outputs, axis=0))
preds_fold = np.vstack(preds_fold)
preds.append(preds_fold)
preds = np.mean(preds, axis=0)
test_df = pd.DataFrame(preds,
columns=[
'epidural', 'intraparenchymal',
'intraventricular', 'subarachnoid', 'subdural',
'any'
])
test_df['ID'] = test_img_paths
# Unpivot table, i.e. wide (N x 6) to long format (6N x 1)
test_df = test_df.melt(id_vars=['ID'])
# Combine the filename column with the variable column
test_df['ID'] = test_df.ID.apply(lambda x: os.path.basename(x).replace(
'.png', '')) + '_' + test_df.variable
test_df['Label'] = test_df['value']
if test_args.hflip:
args.name += '_hflip'
test_df[['ID', 'Label']].to_csv('submissions/%s.csv' % args.name,
index=False)
from datetime import datetime, timedelta
from pathlib import Path
from lib.dataset import Dataset
from client.db_client import DBClient
from client.exchange_client import ExchangeClient
bitmex_exchange_client = ExchangeClient("bitmex", "config.ini")
mysql_client = DBClient("mysql", "config.ini")
dataset_manager = Dataset(mysql_client, bitmex_exchange_client, True)
start_time = datetime.now() - timedelta(days=200)
end_time = datetime.now()
dataset_manager.update_ohlcv("bitmex", start_time)
def main():
parser = argparse.ArgumentParser(description="Generate some useful quantile graphs and perhaps a few histograms too")
parser.add_argument("data_directory", help="the location where we are storing the data.")
parser.add_argument("output_file", help="the file where we will dump the JSON result aggregate data for analysis")
args = parser.parse_args()
not_enough_data = []
cost_increased_types = []
cost_decreased_types = []
def perform_analysis(az, insttype, data_before, data_after):
data_before = PriceTimeSeriesNaive(data_before)
data_after = PriceTimeSeriesNaive(data_after)
days_required = 90
hours_required = days_required * 24
bucket_interval = 350 # bucket interval in seconds
bucket_before = PriceBucketNaive(data_before.expand_with_interval(bucket_interval))
bucket_after = PriceBucketNaive(data_after.expand_with_interval(bucket_interval))
SECONDS_IN_DAY = 3600 * 24
BUCKETS_PER_DAY = SECONDS_IN_DAY / bucket_interval
BUCKETS_PER_HOUR = 3600 / bucket_interval
SIZE_REQUIREMENT = int(BUCKETS_PER_DAY * days_required)
if bucket_before.size() < SIZE_REQUIREMENT or bucket_after.size() < SIZE_REQUIREMENT:
print("\tnot enough data for analysis")
not_enough_data.append((az, insttype))
return
print("\tanalyzing!")
# shrink down to exactly the size requirement
bucket_before = bucket_before.shrink_to_size(SIZE_REQUIREMENT)
bucket_after = bucket_after.shrink_to_size(SIZE_REQUIREMENT)
mean = bucket_before.getAverage()
median = bucket_before.getQuantile(0.5)["price"]
# return the result object containing aggregates of the values we computed
return {
"key": (az, insttype),
"interval_days": days_required, # how many days we are looking at
"total_cost_before": bucket_before.getAverage() * hours_required,
"total_cost_after": bucket_after.getAverage() * hours_required,
"mean_price_before": bucket_before.getAverage(),
"mean_price_after": bucket_after.getAverage(),
"median_price_before": bucket_before.getQuantile(0.5),
"median_price_after": bucket_after.getQuantile(0.5),
}
def transform_query_results(results):
return ({"timestamp": result[0], "price": result[1]} for result in results)
results = []
with Dataset(args.data_directory) as dataset:
count = 0
for az, insttype in dataset.get_databases(): # misleading function name
conn = dataset.open(az, insttype)
# compute the most recent timestamp so we can determine a good
# duration for the intervals we will be examining
c = conn.cursor()
c.execute("SELECT MAX(timestamp) AS timestamp FROM prices")
most_recent_ts = c.fetchone()[0]
interval_duration = most_recent_ts - AFTER_START_EPOCH
if interval_duration < 0: continue
# fetch the data for the before window
c = conn.cursor()
c.execute("SELECT timestamp, price FROM prices "
" WHERE timestamp > %d AND timestamp < %d ORDER BY timestamp"
% (BEFORE_END_EPOCH - interval_duration, BEFORE_END_EPOCH))
data_before = list(transform_query_results(c.fetchall()))
c = conn.cursor()
c.execute("SELECT timestamp, price FROM prices " +
" WHERE timestamp > %d ORDER BY timestamp " % (AFTER_START_EPOCH))
data_after = list(transform_query_results(c.fetchall()))
if len(data_before) == 0 or len(data_after) == 0: continue
print("analyzing data for %s - %s" % (az, insttype))
results.append(
perform_analysis(az, insttype, data_before, data_after)
)
results = list(filter(lambda x: x is not None, results))
results.sort(key=lambda r: r["key"])
with open(args.output_file, "w") as f:
json.dump(results, f)
def main():
args = parse_args()
if args.name is None:
args.name = '%s_%s' % (args.arch, datetime.now().strftime('%m%d%H'))
if not os.path.exists('models/%s' % args.name):
os.makedirs('models/%s' % args.name)
print('Config -----')
for arg in vars(args):
print('- %s: %s' % (arg, getattr(args, arg)))
print('------------')
with open('models/%s/args.txt' % args.name, 'w') as f:
for arg in vars(args):
print('- %s: %s' % (arg, getattr(args, arg)), file=f)
joblib.dump(args, 'models/%s/args.pkl' % args.name)
if args.loss == 'CrossEntropyLoss':
criterion = nn.CrossEntropyLoss().cuda()
elif args.loss == 'FocalLoss':
criterion = FocalLoss().cuda()
elif args.loss == 'MSELoss':
criterion = nn.MSELoss().cuda()
elif args.loss == 'multitask':
criterion = {
'classification': nn.CrossEntropyLoss().cuda(),
'regression': nn.MSELoss().cuda(),
}
else:
raise NotImplementedError
if args.pred_type == 'classification':
num_outputs = 5
elif args.pred_type == 'regression':
num_outputs = 1
elif args.loss == 'multitask':
num_outputs = 6
else:
raise NotImplementedError
cudnn.benchmark = True
model = get_model(model_name=args.arch,
num_outputs=num_outputs,
freeze_bn=args.freeze_bn,
dropout_p=args.dropout_p)
train_transform = []
train_transform = transforms.Compose([
transforms.Resize((args.img_size, args.img_size)),
transforms.RandomAffine(
degrees=(args.rotate_min, args.rotate_max) if args.rotate else 0,
translate=(args.translate_min, args.translate_max) if args.translate else None,
scale=(args.rescale_min, args.rescale_max) if args.rescale else None,
shear=(args.shear_min, args.shear_max) if args.shear else None,
),
transforms.CenterCrop(args.input_size),
transforms.RandomHorizontalFlip(p=0.5 if args.flip else 0),
transforms.RandomVerticalFlip(p=0.5 if args.flip else 0),
transforms.ColorJitter(
brightness=0,
contrast=args.contrast,
saturation=0,
hue=0),
RandomErase(
prob=args.random_erase_prob if args.random_erase else 0,
sl=args.random_erase_sl,
sh=args.random_erase_sh,
r=args.random_erase_r),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
])
val_transform = transforms.Compose([
transforms.Resize((args.img_size, args.input_size)),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
])
# data loading code
if 'diabetic_retinopathy' in args.train_dataset:
diabetic_retinopathy_dir = preprocess(
'diabetic_retinopathy',
args.img_size,
scale=args.scale_radius,
norm=args.normalize,
pad=args.padding,
remove=args.remove)
diabetic_retinopathy_df = pd.read_csv('inputs/diabetic-retinopathy-resized/trainLabels.csv')
diabetic_retinopathy_img_paths = \
diabetic_retinopathy_dir + '/' + diabetic_retinopathy_df['image'].values + '.jpeg'
diabetic_retinopathy_labels = diabetic_retinopathy_df['level'].values
if 'aptos2019' in args.train_dataset:
aptos2019_dir = preprocess(
'aptos2019',
args.img_size,
scale=args.scale_radius,
norm=args.normalize,
pad=args.padding,
remove=args.remove)
aptos2019_df = pd.read_csv('inputs/train.csv')
aptos2019_img_paths = aptos2019_dir + '/' + aptos2019_df['id_code'].values + '.png'
aptos2019_labels = aptos2019_df['diagnosis'].values
if args.train_dataset == 'aptos2019':
skf = StratifiedKFold(n_splits=args.n_splits, shuffle=True, random_state=41)
img_paths = []
labels = []
for fold, (train_idx, val_idx) in enumerate(skf.split(aptos2019_img_paths, aptos2019_labels)):
img_paths.append((aptos2019_img_paths[train_idx], aptos2019_img_paths[val_idx]))
labels.append((aptos2019_labels[train_idx], aptos2019_labels[val_idx]))
elif args.train_dataset == 'diabetic_retinopathy':
img_paths = [(diabetic_retinopathy_img_paths, aptos2019_img_paths)]
labels = [(diabetic_retinopathy_labels, aptos2019_labels)]
elif 'diabetic_retinopathy' in args.train_dataset and 'aptos2019' in args.train_dataset:
skf = StratifiedKFold(n_splits=args.n_splits, shuffle=True, random_state=41)
img_paths = []
labels = []
for fold, (train_idx, val_idx) in enumerate(skf.split(aptos2019_img_paths, aptos2019_labels)):
img_paths.append((np.hstack((aptos2019_img_paths[train_idx], diabetic_retinopathy_img_paths)), aptos2019_img_paths[val_idx]))
labels.append((np.hstack((aptos2019_labels[train_idx], diabetic_retinopathy_labels)), aptos2019_labels[val_idx]))
# else:
# raise NotImplementedError
if args.pseudo_labels:
test_df = pd.read_csv('probs/%s.csv' % args.pseudo_labels)
test_dir = preprocess(
'test',
args.img_size,
scale=args.scale_radius,
norm=args.normalize,
pad=args.padding,
remove=args.remove)
test_img_paths = test_dir + '/' + test_df['id_code'].values + '.png'
test_labels = test_df['diagnosis'].values
for fold in range(len(img_paths)):
img_paths[fold] = (np.hstack((img_paths[fold][0], test_img_paths)), img_paths[fold][1])
labels[fold] = (np.hstack((labels[fold][0], test_labels)), labels[fold][1])
if 'messidor' in args.train_dataset:
test_dir = preprocess(
'messidor',
args.img_size,
scale=args.scale_radius,
norm=args.normalize,
pad=args.padding,
remove=args.remove)
folds = []
best_losses = []
best_scores = []
for fold, ((train_img_paths, val_img_paths), (train_labels, val_labels)) in enumerate(zip(img_paths, labels)):
print('Fold [%d/%d]' %(fold+1, len(img_paths)))
if os.path.exists('models/%s/model_%d.pth' % (args.name, fold+1)):
log = pd.read_csv('models/%s/log_%d.csv' %(args.name, fold+1))
best_loss, best_score = log.loc[log['val_loss'].values.argmin(), ['val_loss', 'val_score']].values
folds.append(str(fold + 1))
best_losses.append(best_loss)
best_scores.append(best_score)
continue
if args.remove_duplicate:
md5_df = pd.read_csv('inputs/strMd5.csv')
duplicate_img_paths = aptos2019_dir + '/' + md5_df[(md5_df.strMd5_count > 1) & (~md5_df.diagnosis.isnull())]['id_code'].values + '.png'
print(duplicate_img_paths)
for duplicate_img_path in duplicate_img_paths:
train_labels = train_labels[train_img_paths != duplicate_img_path]
train_img_paths = train_img_paths[train_img_paths != duplicate_img_path]
val_labels = val_labels[val_img_paths != duplicate_img_path]
val_img_paths = val_img_paths[val_img_paths != duplicate_img_path]
# train
train_set = Dataset(
train_img_paths,
train_labels,
transform=train_transform)
_, class_sample_counts = np.unique(train_labels, return_counts=True)
# print(class_sample_counts)
# weights = 1. / torch.tensor(class_sample_counts, dtype=torch.float)
# weights = np.array([0.2, 0.1, 0.6, 0.1, 0.1])
# samples_weights = weights[train_labels]
# sampler = WeightedRandomSampler(
# weights=samples_weights,
# num_samples=11000,
# replacement=False)
train_loader = torch.utils.data.DataLoader(
train_set,
batch_size=args.batch_size,
shuffle=False if args.class_aware else True,
num_workers=4,
sampler=sampler if args.class_aware else None)
val_set = Dataset(
val_img_paths,
val_labels,
transform=val_transform)
val_loader = torch.utils.data.DataLoader(
val_set,
batch_size=args.batch_size,
shuffle=False,
num_workers=4)
# create model
model = get_model(model_name=args.arch,
num_outputs=num_outputs,
freeze_bn=args.freeze_bn,
dropout_p=args.dropout_p)
model = model.cuda()
if args.pretrained_model is not None:
model.load_state_dict(torch.load('models/%s/model_%d.pth' % (args.pretrained_model, fold+1)))
# print(model)
if args.optimizer == 'Adam':
optimizer = optim.Adam(
filter(lambda p: p.requires_grad, model.parameters()), lr=args.lr)
elif args.optimizer == 'AdamW':
optimizer = optim.AdamW(
filter(lambda p: p.requires_grad, model.parameters()), lr=args.lr)
elif args.optimizer == 'RAdam':
optimizer = RAdam(
filter(lambda p: p.requires_grad, model.parameters()), lr=args.lr)
elif args.optimizer == 'SGD':
optimizer = optim.SGD(filter(lambda p: p.requires_grad, model.parameters()), lr=args.lr,
momentum=args.momentum, weight_decay=args.weight_decay, nesterov=args.nesterov)
if args.scheduler == 'CosineAnnealingLR':
scheduler = lr_scheduler.CosineAnnealingLR(
optimizer, T_max=args.epochs, eta_min=args.min_lr)
elif args.scheduler == 'ReduceLROnPlateau':
scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, factor=args.factor, patience=args.patience,
verbose=1, min_lr=args.min_lr)
log = pd.DataFrame(index=[], columns=[
'epoch', 'loss', 'score', 'val_loss', 'val_score'
])
log = {
'epoch': [],
'loss': [],
'score': [],
'val_loss': [],
'val_score': [],
}
best_loss = float('inf')
best_score = 0
for epoch in range(args.epochs):
print('Epoch [%d/%d]' % (epoch + 1, args.epochs))
# train for one epoch
train_loss, train_score = train(
args, train_loader, model, criterion, optimizer, epoch)
# evaluate on validation set
val_loss, val_score = validate(args, val_loader, model, criterion)
if args.scheduler == 'CosineAnnealingLR':
scheduler.step()
elif args.scheduler == 'ReduceLROnPlateau':
scheduler.step(val_loss)
print('loss %.4f - score %.4f - val_loss %.4f - val_score %.4f'
% (train_loss, train_score, val_loss, val_score))
log['epoch'].append(epoch)
log['loss'].append(train_loss)
log['score'].append(train_score)
log['val_loss'].append(val_loss)
log['val_score'].append(val_score)
pd.DataFrame(log).to_csv('models/%s/log_%d.csv' % (args.name, fold+1), index=False)
if val_loss < best_loss:
torch.save(model.state_dict(), 'models/%s/model_%d.pth' % (args.name, fold+1))
best_loss = val_loss
best_score = val_score
print("=> saved best model")
print('val_loss: %f' % best_loss)
print('val_score: %f' % best_score)
folds.append(str(fold + 1))
best_losses.append(best_loss)
best_scores.append(best_score)
results = pd.DataFrame({
'fold': folds + ['mean'],
'best_loss': best_losses + [np.mean(best_losses)],
'best_score': best_scores + [np.mean(best_scores)],
})
print(results)
results.to_csv('models/%s/results.csv' % args.name, index=False)
torch.cuda.empty_cache()
if not args.cv:
break
def main():
args, args_text = _parse_args()
seed = args.seed
torch.manual_seed(seed)
torch.cuda.manual_seed_all(seed)
np.random.seed(seed)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
if args.local_rank == 0:
print("rank:{0},word_size:{1},dist_url:{2}".format(
local_rank, word_size, dist_url))
assert args.model_selection in [14, 114, 470, 600, 285, 42]
if args.model_selection == 470:
arch_list = [[0], [3, 4, 3, 1], [3, 2, 3, 0], [3, 3, 3, 1],
[3, 3, 3, 3], [3, 3, 3, 3], [0]]
arch_def = [
# stage 0, 112x112 in
['ds_r1_k3_s1_e1_c16_se0.25'],
# stage 1, 112x112 in
[
'ir_r1_k3_s2_e4_c24_se0.25', 'ir_r1_k3_s1_e4_c24_se0.25',
'ir_r1_k3_s1_e4_c24_se0.25', 'ir_r1_k3_s1_e4_c24_se0.25'
],
# stage 2, 56x56 in
[
'ir_r1_k5_s2_e4_c40_se0.25', 'ir_r1_k5_s1_e4_c40_se0.25',
'ir_r1_k5_s2_e4_c40_se0.25', 'ir_r1_k5_s2_e4_c40_se0.25'
],
# stage 3, 28x28 in
[
'ir_r1_k3_s2_e6_c80_se0.25', 'ir_r1_k3_s1_e4_c80_se0.25',
'ir_r1_k3_s1_e4_c80_se0.25', 'ir_r2_k3_s1_e4_c80_se0.25'
],
# stage 4, 14x14in
[
'ir_r1_k3_s1_e6_c96_se0.25', 'ir_r1_k3_s1_e6_c96_se0.25',
'ir_r1_k3_s1_e6_c96_se0.25', 'ir_r1_k3_s1_e6_c96_se0.25'
],
# stage 5, 14x14in
[
'ir_r1_k5_s2_e6_c192_se0.25', 'ir_r1_k5_s1_e6_c192_se0.25',
'ir_r1_k5_s2_e6_c192_se0.25', 'ir_r1_k5_s2_e6_c192_se0.25'
],
# stage 6, 7x7 in
['cn_r1_k1_s1_c320_se0.25'],
]
args.img_size = 224
elif args.model_selection == 42:
arch_list = [[0], [3], [3, 1], [3, 1], [3, 3, 3], [3, 3], [0]]
arch_def = [
# stage 0, 112x112 in
['ds_r1_k3_s1_e1_c16_se0.25'],
# stage 1, 112x112 in
['ir_r1_k3_s2_e4_c24_se0.25'],
# stage 2, 56x56 in
['ir_r1_k5_s2_e4_c40_se0.25', 'ir_r1_k5_s2_e4_c40_se0.25'],
# stage 3, 28x28 in
['ir_r1_k3_s2_e6_c80_se0.25', 'ir_r1_k3_s2_e6_c80_se0.25'],
# stage 4, 14x14in
[
'ir_r1_k3_s1_e6_c96_se0.25', 'ir_r1_k3_s1_e6_c96_se0.25',
'ir_r1_k3_s1_e6_c96_se0.25'
],
# stage 5, 14x14in
['ir_r1_k5_s2_e6_c192_se0.25', 'ir_r1_k5_s2_e6_c192_se0.25'],
# stage 6, 7x7 in
['cn_r1_k1_s1_c320_se0.25'],
]
args.img_size = 96
elif args.model_selection == 14:
arch_list = [[0], [3], [3, 3], [3, 3], [3], [3], [0]]
arch_def = [
# stage 0, 112x112 in
['ds_r1_k3_s1_e1_c16_se0.25'],
# stage 1, 112x112 in
['ir_r1_k3_s2_e4_c24_se0.25'],
# stage 2, 56x56 in
['ir_r1_k5_s2_e4_c40_se0.25', 'ir_r1_k3_s2_e4_c40_se0.25'],
# stage 3, 28x28 in
['ir_r1_k3_s2_e6_c80_se0.25', 'ir_r1_k3_s2_e4_c80_se0.25'],
# stage 4, 14x14in
['ir_r1_k3_s1_e6_c96_se0.25'],
# stage 5, 14x14in
['ir_r1_k5_s2_e6_c192_se0.25'],
# stage 6, 7x7 in
['cn_r1_k1_s1_c320_se0.25'],
]
args.img_size = 64
elif args.model_selection == 112:
arch_list = [[0], [3], [3, 3], [3, 3], [3, 3, 3], [3, 3], [0]]
arch_def = [
# stage 0, 112x112 in
['ds_r1_k3_s1_e1_c16_se0.25'],
# stage 1, 112x112 in
['ir_r1_k3_s2_e4_c24_se0.25'],
# stage 2, 56x56 in
['ir_r1_k5_s2_e4_c40_se0.25', 'ir_r1_k3_s2_e4_c40_se0.25'],
# stage 3, 28x28 in
['ir_r1_k3_s2_e6_c80_se0.25', 'ir_r1_k3_s2_e6_c80_se0.25'],
# stage 4, 14x14in
[
'ir_r1_k3_s1_e6_c96_se0.25', 'ir_r1_k3_s1_e6_c96_se0.25',
'ir_r1_k3_s1_e6_c96_se0.25'
],
# stage 5, 14x14in
['ir_r1_k5_s2_e6_c192_se0.25', 'ir_r1_k5_s2_e6_c192_se0.25'],
# stage 6, 7x7 in
['cn_r1_k1_s1_c320_se0.25'],
]
args.img_size = 160
elif args.model_selection == 285:
arch_list = [[0], [3], [3, 3], [3, 1, 3], [3, 3, 3, 3], [3, 3, 3], [0]]
arch_def = [
# stage 0, 112x112 in
['ds_r1_k3_s1_e1_c16_se0.25'],
# stage 1, 112x112 in
['ir_r1_k3_s2_e4_c24_se0.25'],
# stage 2, 56x56 in
['ir_r1_k5_s2_e4_c40_se0.25', 'ir_r1_k5_s2_e4_c40_se0.25'],
# stage 3, 28x28 in
[
'ir_r1_k3_s2_e6_c80_se0.25', 'ir_r1_k3_s2_e6_c80_se0.25',
'ir_r1_k3_s2_e6_c80_se0.25'
],
# stage 4, 14x14in
[
'ir_r1_k3_s1_e6_c96_se0.25', 'ir_r1_k3_s1_e6_c96_se0.25',
'ir_r1_k3_s1_e6_c96_se0.25', 'ir_r1_k3_s1_e6_c96_se0.25'
],
# stage 5, 14x14in
[
'ir_r1_k5_s2_e6_c192_se0.25', 'ir_r1_k5_s2_e6_c192_se0.25',
'ir_r1_k5_s2_e6_c192_se0.25'
],
# stage 6, 7x7 in
['cn_r1_k1_s1_c320_se0.25'],
]
args.img_size = 224
elif args.model_selection == 600:
arch_list = [[0], [3, 3, 2, 3, 3], [3, 2, 3, 2, 3], [3, 2, 3, 2, 3],
[3, 3, 2, 2, 3, 3], [3, 3, 2, 3, 3, 3], [0]]
arch_def = [
# stage 0, 112x112 in
['ds_r1_k3_s1_e1_c16_se0.25'],
# stage 1, 112x112 in
[
'ir_r1_k3_s2_e4_c24_se0.25', 'ir_r1_k3_s2_e4_c24_se0.25',
'ir_r1_k3_s2_e4_c24_se0.25', 'ir_r1_k3_s2_e4_c24_se0.25',
'ir_r1_k3_s2_e4_c24_se0.25'
],
# stage 2, 56x56 in
[
'ir_r1_k5_s2_e4_c40_se0.25', 'ir_r1_k5_s2_e4_c40_se0.25',
'ir_r1_k5_s2_e4_c40_se0.25', 'ir_r1_k5_s2_e4_c40_se0.25',
'ir_r1_k5_s2_e4_c40_se0.25'
],
# stage 3, 28x28 in
[
'ir_r1_k3_s2_e6_c80_se0.25', 'ir_r1_k3_s1_e4_c80_se0.25',
'ir_r1_k3_s1_e4_c80_se0.25', 'ir_r1_k3_s1_e4_c80_se0.25',
'ir_r1_k3_s1_e4_c80_se0.25'
],
# stage 4, 14x14in
[
'ir_r1_k3_s1_e6_c96_se0.25', 'ir_r1_k3_s1_e6_c96_se0.25',
'ir_r1_k3_s1_e6_c96_se0.25', 'ir_r1_k3_s1_e6_c96_se0.25',
'ir_r1_k3_s1_e6_c96_se0.25', 'ir_r1_k3_s1_e6_c96_se0.25'
],
# stage 5, 14x14in
[
'ir_r1_k5_s2_e6_c192_se0.25', 'ir_r1_k5_s1_e6_c192_se0.25',
'ir_r1_k5_s1_e6_c192_se0.25', 'ir_r1_k5_s1_e6_c192_se0.25',
'ir_r1_k5_s1_e6_c192_se0.25', 'ir_r1_k5_s1_e6_c192_se0.25'
],
# stage 6, 7x7 in
['cn_r1_k1_s1_c320_se0.25'],
]
args.img_size = 224
model = _gen_childnet(arch_list,
arch_def,
num_classes=args.num_classes,
drop_rate=args.drop,
drop_path_rate=args.drop_path,
global_pool=args.gp,
bn_momentum=args.bn_momentum,
bn_eps=args.bn_eps,
pool_bn=args.pool_bn,
zero_gamma=args.zero_gamma)
data_config = resolve_data_config(vars(args),
model=model,
verbose=args.local_rank == 0)
if args.local_rank == 0:
img_size = args.img_size or 224
scope(model, input_size=(3, img_size, img_size))
eval_metric = args.eval_metric
best_metric = None
best_epoch = None
saver = None
output_dir = ''
if args.local_rank == 0:
output_base = args.output if args.output else './experiments/'
exp_name = '-'.join([
args.name,
datetime.now().strftime("%Y%m%d-%H%M%S"), args.model,
str(data_config['input_size'][-1])
])
output_dir = get_outdir(output_base, 'test', exp_name)
logger = get_logger(os.path.join(output_dir, 'test.log'))
writer = SummaryWriter(os.path.join(output_dir, 'runs'))
decreasing = True if eval_metric == 'loss' else False
if not args.nosave:
saver = CheckpointSaver(checkpoint_dir=output_dir,
decreasing=decreasing)
with open(os.path.join(output_dir, 'config.yaml'), 'w') as f:
f.write(args_text)
else:
writer = None
logger = None
args.prefetcher = not args.no_prefetcher
args.distributed = False
if 'WORLD_SIZE' in os.environ:
args.distributed = int(os.environ['WORLD_SIZE']) > 1
if args.distributed and args.num_gpu > 1 and args.local_rank == 0:
logger.warning(
'Using more than one GPU per process in distributed mode is not allowed. Setting num_gpu to 1.'
)
args.num_gpu = 1
args.device = 'cuda:0'
args.world_size = 1
args.rank = 0 # global rank
args.distributed = True
if args.distributed:
args.num_gpu = 1
args.device = 'cuda:%d' % args.local_rank
torch.cuda.set_device(args.local_rank)
torch.distributed.init_process_group(backend='nccl',
init_method='env://')
args.world_size = torch.distributed.get_world_size()
args.rank = torch.distributed.get_rank()
assert args.rank >= 0
if args.local_rank == 0:
if args.distributed:
logger.info(
'Training in distributed mode with multiple processes, 1 GPU per process. Process %d, total %d.'
% (args.rank, args.world_size))
else:
logger.info('Training with a single process on %d GPUs.' %
args.num_gpu)
num_aug_splits = 0
if args.aug_splits > 0:
assert args.aug_splits > 1, 'A split of 1 makes no sense'
num_aug_splits = args.aug_splits
if args.split_bn:
assert num_aug_splits > 1 or args.resplit
model = convert_splitbn_model(model, max(num_aug_splits, 2))
if os.path.exists(args.initial_checkpoint):
load_checkpoint(model, args.initial_checkpoint)
if args.local_rank == 0:
logger.info('Model %s created, param count: %d' %
(args.model, sum([m.numel() for m in model.parameters()])))
if args.num_gpu > 1:
if args.amp:
if args.local_rank == 0:
logger.warning(
'AMP does not work well with nn.DataParallel, disabling. Use distributed mode for multi-GPU AMP.'
)
args.amp = False
model = nn.DataParallel(model,
device_ids=list(range(args.num_gpu))).cuda()
else:
model.cuda()
optimizer = create_optimizer(args, model)
use_amp = False
if has_apex and args.amp:
model, optimizer = amp.initialize(model, optimizer, opt_level='O1')
use_amp = True
if args.local_rank == 0:
logger.info('NVIDIA APEX {}. '
' {}.'.format('installed' if has_apex else 'not installed',
'on' if use_amp else 'off'))
# optionally resume from a checkpoint
resume_state = {}
resume_epoch = None
if args.resume:
resume_state, resume_epoch = resume_checkpoint(model, args.resume)
if resume_state and not args.no_resume_opt:
if 'optimizer' in resume_state:
if args.local_rank == 0:
logging.info('Restoring Optimizer state from checkpoint')
optimizer.load_state_dict(resume_state['optimizer'])
if use_amp and 'amp' in resume_state and 'load_state_dict' in amp.__dict__:
if args.local_rank == 0:
logging.info('Restoring NVIDIA AMP state from checkpoint')
amp.load_state_dict(resume_state['amp'])
del resume_state
model_ema = None
if args.model_ema:
# Important to create EMA model after cuda(), DP wrapper, and AMP but before SyncBN and DDP wrapper
model_ema = ModelEma(model,
decay=args.model_ema_decay,
device='cpu' if args.model_ema_force_cpu else '',
resume=args.resume)
if args.distributed:
if args.sync_bn:
assert not args.split_bn
try:
if has_apex:
model = convert_syncbn_model(model)
else:
model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(
model)
if args.local_rank == 0:
logger.info(
'Converted model to use Synchronized BatchNorm.')
except Exception as e:
if args.local_rank == 0:
logger.error(
'Failed to enable Synchronized BatchNorm. Install Apex or Torch >= 1.1'
)
if has_apex:
model = DDP(model, delay_allreduce=True)
else:
if args.local_rank == 0:
logger.info(
"Using torch DistributedDataParallel. Install NVIDIA Apex for Apex DDP."
)
model = DDP(model,
device_ids=[args.local_rank
]) # can use device str in Torch >= 1.1
# NOTE: EMA model does not need to be wrapped by DDP
eval_dir = os.path.join(args.data, 'val')
if not os.path.exists(eval_dir) and args.local_rank == 0:
logger.error(
'Validation folder does not exist at: {}'.format(eval_dir))
exit(1)
dataset_eval = Dataset(eval_dir)
loader_eval = create_loader(
dataset_eval,
input_size=data_config['input_size'],
batch_size=args.validation_batch_size_multiplier * args.batch_size,
is_training=False,
use_prefetcher=args.prefetcher,
interpolation=data_config['interpolation'],
mean=data_config['mean'],
std=data_config['std'],
num_workers=args.workers,
distributed=args.distributed,
crop_pct=data_config['crop_pct'],
pin_memory=args.pin_mem,
)
validate_loss_fn = nn.CrossEntropyLoss().cuda()
validate(0,
model_ema.ema,
loader_eval,
validate_loss_fn,
args,
log_suffix=' (EMA)',
logger=logger,
writer=writer)
from client.exchange_client import ExchangeClient
from client.db_client import DBClient
from client.exchange_ws_client import WSClient
from lib.pandamex import PandaMex
from lib.time_ms import TimeMS
from lib.dataset import Dataset
from hypothesis_test.fetch_high_frequency_data_test import FetchHighFrequencyData
from hypothesis_test.h_price_move_probability import HPriceMovePlobability
from hypothesis_test.volatility_dependent_offset_test import VolatilityDependentOffsetTest
from bot.bottom_trend_follower import BottomTrendFollow
bitmex_exchange_client = ExchangeClient("bitmex",
Path("tradingbot/config.ini"))
# influx_client = DBClient("influxdb")
# print(influx_client.connector.get_list_database())
mysql_client = DBClient("mysql", Path("tradingbot/config.ini"))
# update database
dataset_manager = Dataset(mysql_client, bitmex_exchange_client)
download_start_time = datetime.now() - timedelta(days=1000)
dataset_manager.update_ohlcv("bitmex", download_start_time)
bot_bot = BottomTrendFollow(db_client=mysql_client,
exchange_client=bitmex_exchange_client,
is_backtest=True)
bot_bot.run()
class UVInpainting():
def __init__(self, config, device, sess=None, graph=None):
'''
if game_lm is not None, the result (mesh obj and UV texture map)
will be convert from nsh to the game
'''
self.config = config
self.name = config.name
self.device = device
self.sess = sess
self.graph = graph
self.log = logging.getLogger('x')
self.rot_order = 'XYZ'
self.debug = config.debug
self.ex_idx = [4, 5, 8]
self.inpaint_model = InpaintingModel(config,
device,
self.rot_order,
debug=self.debug).to(device)
# self.inpaint_model = InpaintingModel(config, device, self.debug)
self.epoch = 0
if config.restore:
self.epoch = self.inpaint_model.load()
# self.phase = config.phase
if config.mode == 'train':
num_test = 2048
flist = glob(os.path.join(config.data_dir, '*_uv.png'))
random.shuffle(flist)
train_flist = flist[:-2 * num_test]
val_flist = flist[-2 * num_test:-num_test]
test_flist = flist[-num_test:]
num_test = 300
flist_gt = glob(os.path.join(config.data_gt_dir, '*_uv*.png'))
random.shuffle(flist_gt)
train_flist_gt = flist_gt[:-2 * num_test]
val_flist_gt = flist_gt[-2 * num_test:-num_test]
test_flist_gt = flist_gt[-num_test:]
self.train_dataset = Dataset(config, train_flist_gt, train_flist)
self.val_dataset = Dataset(config, val_flist_gt, val_flist)
self.val_sample_iterator = self.val_dataset.create_iterator(
config.batch_size)
self.test_dataset = Dataset(config,
test_flist_gt,
test_flist,
test=True)
self.test_sample_iterator = self.test_dataset.create_iterator(
config.batch_size)
self.samples_dir = os.path.join('samples', config.name)
os.makedirs(self.samples_dir, exist_ok=True)
elif config.mode == 'test':
self.test_dataset = Dataset(config, [], [], test=True)
self.init_test()
def train(self):
train_loader = DataLoader(dataset=self.train_dataset,
batch_size=self.config.batch_size,
num_workers=self.config.workers,
drop_last=True,
shuffle=True)
if not self.train_dataset:
self.log.info('No training data was provided!')
return
writer = SummaryWriter('logs/' + self.config.name)
while self.epoch < self.config.epochs:
self.log.info('Training epoch: %d', self.epoch)
self.epoch += 1
for items in train_loader:
self.inpaint_model.train()
iteration = self.inpaint_model.iteration
images, uvmaps, uvmap_gts, vertices, coeffs, rand_images, rand_uvmaps, rand_verts, rand_coeffs = self.to_device(
*items)
_, gen_loss, im_dis_loss, uv_dis_loss, logs = self.inpaint_model.process(
images, uvmaps, uvmap_gts, vertices, coeffs)
for k, v in logs.items():
writer.add_scalar(k, v, iteration)
self.inpaint_model.backward(gen_loss=gen_loss,
im_dis_loss=im_dis_loss,
uv_dis_loss=uv_dis_loss)
_, rand_gen_loss, rand_im_dis_loss, rand_uv_dis_loss, rand_logs = self.inpaint_model.process(
rand_images, rand_uvmaps, uvmap_gts, rand_verts,
rand_coeffs, False)
self.inpaint_model.backward(gen_loss=rand_gen_loss,
im_dis_loss=rand_im_dis_loss,
uv_dis_loss=rand_uv_dis_loss)
self.inpaint_model.iteration += 1
# log model at checkpoints
if self.config.log_interval and iteration % self.config.log_interval == 0:
info = 'Epoch: {} Iter:{}\n'.format(self.epoch, iteration)
info = create_log(logs, info)
self.log.info(info)
info = 'Epoch: {} Iter:{} RANDOM UVMAP\n'.format(
self.epoch, iteration)
info = create_log(rand_logs, info)
self.log.info(info)
# sample model at checkpoints
if self.config.sample_interval and iteration % self.config.sample_interval == 0:
self.val_sample()
self.test_sample()
if self.config.ckpt_interval and iteration % self.config.ckpt_interval == 0:
self.inpaint_model.save(self.epoch)
self.log.info('\nEnd training....')
def val_sample(self, it=None):
self.inpaint_model.eval()
val_items = next(self.val_sample_iterator)
images, uvmaps, uvmap_gts, vertices, coeffs, rand_images, rand_uvmaps, rand_verts, rand_coeffs = self.to_device(
*val_items)
gen_uvmaps, im_merged = self.sample(images, uvmaps, vertices, coeffs)
rand_gen_uvmaps, rand_im_merged = self.sample(rand_images, rand_uvmaps,
rand_verts, rand_coeffs)
iteration = self.inpaint_model.iteration
if it is not None:
iteration = it
image_per_row = 2
if self.config.batch_size <= 6:
image_per_row = 1
images = utils.stitch_images(
utils.to_uint8_torch(images[:, :3]),
utils.to_uint8_torch(uvmaps[:, :3]),
utils.to_uint8_torch(gen_uvmaps[:, :3]),
utils.to_uint8_torch(uvmap_gts),
utils.to_uint8_torch(im_merged[:self.config.batch_size]),
utils.to_uint8_torch(im_merged[self.config.batch_size:]),
im_size=self.config.uv_size,
img_per_row=image_per_row)
name = os.path.join(self.samples_dir,
str(iteration - 1).zfill(5) + ".png")
images.save(name)
self.log.info('Val Sample saved to %s', name)
images = utils.stitch_images(
utils.to_uint8_torch(rand_images[:, :3]),
utils.to_uint8_torch(rand_uvmaps[:, :3]),
utils.to_uint8_torch(rand_gen_uvmaps[:, :3]),
utils.to_uint8_torch(rand_im_merged[:self.config.batch_size]),
utils.to_uint8_torch(rand_im_merged[self.config.batch_size:]),
im_size=self.config.uv_size,
img_per_row=image_per_row)
name = os.path.join(self.samples_dir,
str(iteration - 1).zfill(5) + "_r.png")
images.save(name)
self.log.info('Val Sample saved to %s', name)
def test_sample(self, it=None):
self.inpaint_model.eval()
test_items = next(self.test_sample_iterator)
images, uvmaps, vertices, coeffs = self.to_device(*test_items)
gen_uvmaps, im_merged = self.sample(images, uvmaps, vertices, coeffs)
iteration = self.inpaint_model.iteration
if it is not None:
iteration = it
image_per_row = 2
if self.config.batch_size <= 6:
image_per_row = 1
images = utils.stitch_images(
utils.to_uint8_torch(images[:, :3]),
utils.to_uint8_torch(uvmaps[:, :3]),
utils.to_uint8_torch(gen_uvmaps[:, :3]),
utils.to_uint8_torch(im_merged[:self.config.batch_size]),
utils.to_uint8_torch(im_merged[self.config.batch_size:]),
im_size=self.config.uv_size,
img_per_row=image_per_row)
# path = os.path.join(self.samples_dir, self.name)
name = os.path.join(self.samples_dir,
str(iteration - 1).zfill(5) + "_t.png")
os.makedirs(self.samples_dir, exist_ok=True)
images.save(name)
self.log.info('Test sample saved to %s\n', name)
def sample(self, images, uvmaps, vertices, coeffs):
gen_uvmaps, renders, _ = self.inpaint_model(images[:, :3],
uvmaps,
vertices,
coeffs,
fix_uv=True)
# io.imsave('tmp/render.png', renders[0].permute(1,2,0).cpu().detach().numpy())
double_images = torch.cat([images, torch.flip(images, (3, ))], dim=0)
no_l_eye = double_images[:, -1:] != self.ex_idx[0]
no_r_eye = double_images[:, -1:] != self.ex_idx[1]
no_mouth = double_images[:, -1:] != self.ex_idx[2]
mask = renders[:, 3:4] * no_l_eye.float() * no_r_eye.float(
) * no_mouth.float()
im_merged = double_images[:, :3] * (1 - mask) + renders[:, :3] * mask
# io.imsave('tmp/mask.png', mask[0, 0].cpu().detach().numpy())
return gen_uvmaps.cpu(), im_merged.cpu()
def init_test(self):
self.segmenter = Segment(self.device)
up_line = 100
bt_line = 80
self.transfers = {}
self.uv_creators = {}
self.nsh_face_tris = {}
self.nsh_meshes = {}
self.nsh_face_meshes = {}
for face_model in ['230']:
self.transfers[face_model] = Shape_Transfer(face_model=face_model,
device=self.device)
self.uv_creators[face_model] = UVCreator(
face_model=face_model,
bfm_version=self.config.bfm_version,
device=self.device)
self.nsh_face_meshes[face_model] = meshio.Mesh(
'data/mesh/{}/nsh_bfm_face.obj'.format(face_model))
self.nsh_face_tris[face_model] = self.to_tensor(
self.nsh_face_meshes[face_model].triangles, torch.int64)
self.nsh_meshes[face_model] = meshio.Mesh(
'data/mesh/{}/nsh_std.obj'.format(face_model), group=True)
self.up_line = int(up_line * (self.config.uv_size / 1024))
self.bt_line = int(bt_line * (self.config.uv_size / 1024))
self.eye_lm_idx = np.loadtxt('data/mesh/eye_lm_idx.txt',
dtype=np.int32)
self.cropper = ImageCropper(self.config.im_size, use_dlib=False)
self.reconstructor = Deep3DFace(self.sess, self.graph)
R, T = look_at_view_transform(10, 0, 0)
self.cameras = OpenGLPerspectiveCameras(znear=0.001,
zfar=30.0,
aspect_ratio=1.0,
fov=12.5936,
degrees=True,
R=R,
T=T,
device=self.device)
raster_settings = RasterizationSettings(image_size=512,
blur_radius=0.0,
faces_per_pixel=1,
bin_size=0,
cull_backfaces=True)
self.rasterizer = MeshRasterizer(cameras=self.cameras,
raster_settings=raster_settings)
def preprocess(self, image, face_model):
#* input image should be uint8, in RGB order
image = utils.center_crop_resize(image, self.config.im_size)
image = self.cropper.crop_image(image, self.config.im_size)
image = image[:, ::-1].copy()
images_224 = cv2.resize(image, (224, 224),
interpolation=cv2.INTER_AREA).astype(
np.float32)[None]
images = self.to_tensor(image[None])
segments = self.segmenter.segment_torch(images)
segments = center_crop(segments, images.shape[1])
image_segment = torch.cat([images, segments[..., None]], dim=-1)
image_segment = image_segment.permute(0, 3, 1, 2)
coeff, bfm_vert, bfm_neu_vert = self.reconstructor.predict(
images_224, neutral=True)
bfm_neu_vert = self.to_tensor(bfm_neu_vert)
#! using torch from now on -----------------------------
bfm_vert = self.to_tensor(bfm_vert)
nsh_vert = self.transfers[face_model].transfer_shape_torch(bfm_vert)
nsh_neu_vert = None
nsh_neu_vert = self.transfers[face_model].transfer_shape_torch(
bfm_neu_vert)
nsh_face_vert = nsh_vert[self.uv_creators[face_model].
nsh_face_start_idx:]
coeff = self.to_tensor(coeff[None])
_, _, _, angles, _, translation = utils.split_bfm09_coeff(coeff)
# angle = (angle / 180.0 * math.pi) if degrees else angle
transformer = Transform3d(device=self.device)
transformer = transformer.rotate_axis_angle(angles[:, 0],
self.rot_order[0], False)
transformer = transformer.rotate_axis_angle(angles[:, 1],
self.rot_order[1], False)
transformer = transformer.rotate_axis_angle(angles[:, 2],
self.rot_order[2], False)
transformer = transformer.translate(translation)
nsh_trans_vert = transformer.transform_points(nsh_face_vert[None])
nsh_shift_vert = nsh_trans_vert[0] - self.to_tensor([[0, 0, 10]])
image_segment = torch.flip(image_segment, (3, )).type(torch.float32)
nsh_trans_mesh = Meshes(nsh_trans_vert,
self.nsh_face_tris[face_model][None])
fragment = self.rasterizer(nsh_trans_mesh)
visible_face = torch.unique(
fragment.pix_to_face)[1:] # exclude face id -1
visible_vert = self.nsh_face_tris[face_model][visible_face]
visible_vert = torch.unique(visible_vert)
vert_alpha = torch.zeros([nsh_shift_vert.shape[0], 1],
device=self.device)
vert_alpha[visible_vert] = 1
nsh_shift_vert_alpha = torch.cat([nsh_shift_vert, vert_alpha], axis=-1)
uvmap = self.uv_creators[face_model].create_nsh_uv_torch(
nsh_shift_vert_alpha, image_segment, self.config.uv_size)
uvmap[..., 3] = uvmap[..., 3] + uvmap[..., 4] * 128
uvmap = uvmap[..., :4].cpu().numpy()
uvmap = self.test_dataset.process_uvmap(uvmap.astype(np.uint8),
dark_brow=True)
images = images.permute(0, 3, 1, 2) / 127.5 - 1.0
images = F.interpolate(images,
size=self.config.im_size,
mode='bilinear',
align_corners=False)
segments = F.interpolate(segments[:, None],
size=self.config.im_size,
mode='nearest')
images = torch.cat([images, segments], dim=1)
uvmaps = uvmap[None].permute(0, 3, 1, 2)
return images, uvmaps, coeff, nsh_face_vert, nsh_neu_vert
def predict(self,
image,
out_dir,
idx=None,
deploy=False,
face_model='230'):
'''deploy for nsh'''
if not deploy and idx is None:
idx = '{:>05d}'.format(idx)
images, uvmaps, params, nsh_face_vert, nsh_neu_vert = self.preprocess(
image, face_model)
fnames = []
gen_uvmaps = self.inpaint_model.forward(images[:, :3],
uvmaps,
nsh_face_vert[None],
params,
fix_uv=True,
deploy=deploy,
face_model=face_model)
nsh_uv = F.interpolate(gen_uvmaps.detach(),
size=1024,
mode='bilinear',
align_corners=False)[0]
fnames.append(os.path.join(out_dir, '{}_uv.png'.format(idx)))
self.imsave(fnames[-1], nsh_uv, False, True)
lm_idx = self.to_tensor(self.transfers[face_model].lm_icp_idx,
torch.int64)
nsh_vert_lm = nsh_neu_vert[None, lm_idx]
nsh_std_lm = self.to_tensor(
self.transfers[face_model].tgt_std_vert)[None, lm_idx]
R, T, s = corresponding_points_alignment(nsh_vert_lm,
nsh_std_lm,
estimate_scale=True)
s = s * 0.97
nsh_neu_vert_trans = (
s[:, None, None] * torch.bmm(nsh_neu_vert[None], R) +
T[:, None, :])[0]
nsh_neu_vert = nsh_neu_vert_trans.cpu().numpy()
nsh_neu_vert = self.transfers[face_model].normalize(nsh_neu_vert)
fnames.append(os.path.join(out_dir, '{}_neu.obj'.format(idx)))
meshio.write_obj(
fnames[-1],
nsh_neu_vert[self.uv_creators[face_model].nsh_face_start_idx:],
self.nsh_face_meshes[face_model].triangles,
texcoords=self.nsh_face_meshes[face_model].texcoords,
mtllib=True,
uv_name='{}_uv'.format(idx))
fnames.append(os.path.join(out_dir, '{}_neu.mtl'.format(idx)))
try:
self.imsave(os.path.join(out_dir, '{}_input.jpg'.format(idx)),
images[0, :3], True)
except:
pass
def to_device(self, *args):
return (item.to(self.device) for item in args)
def to_tensor(self, array, dtype=torch.float32):
if not isinstance(array, np.ndarray):
array = np.array(array)
return torch.from_numpy(array).type(dtype).to(self.device)
def imsave(self, path, image, h_flip=False, v_flip=False):
image = utils.to_uint8_torch(image.cpu()).numpy()
if h_flip:
image = image[:, ::-1]
if v_flip:
image = image[::-1]
io.imsave(path, image)
def compute_eye_param(self, vertices, eye_lm_idx, face_model):
nsh_vert_lm = vertices[None, eye_lm_idx]
nsh_std_lm = self.to_tensor(
self.transfers[face_model].tgt_std_vert)[None, eye_lm_idx]
R, T, s = corresponding_points_alignment(nsh_vert_lm,
nsh_std_lm,
estimate_scale=True)
R = R.cpu().numpy()[0]
T = T.cpu().numpy()[0]
s = s.cpu().numpy()
angle = Rotation.from_matrix(R).as_euler('xyz')
eye_param = np.concatenate([angle, T, s])
return eye_param
from technical_analysis.roc import TechnicalAnalysisROC
from technical_analysis.rsi import TechnicalAnalysisRSI
from technical_analysis.so import TechnicalAnalysisSTOCH
from technical_analysis.williamsr import TechnicalAnalysisWilliamsR
from technical_analysis.wma import TechnicalAnalysisWMA
# option settings
pd.set_option("display.max_columns", 250)
pd.set_option("display.max_rows", 250)
import warnings
warnings.filterwarnings('ignore')
plt.rcParams['figure.figsize'] = (10.0, 20.0)
bitmex_exchange_client = ExchangeClient("bitmex", Path(config_ini))
mysql_client = DBClient("mysql", Path(config_ini))
dataset_manager = Dataset(mysql_client, bitmex_exchange_client, True)
dataset_manager.update_ohlcv("bitmex",
start_time=datetime.now() - timedelta(days=365),
with_ta=True)
# manually added
def get_joined_params_and_summary():
bot = BottomTrendFollow(db_client=mysql_client,
exchange_client=bitmex_exchange_client,
is_backtest=True)
backtest_management = bot.trading_bot_backtest.trading_bot_backtest_db.backtest_management_table(
)
backtest_summary = BacktestSummary()
query_management = "SELECT * FROM " + bot.trading_bot_backtest.trading_bot_backtest_db.backtest_management_table_name + ";"
from technical_analysis.roc import TechnicalAnalysisROC
from technical_analysis.rsi import TechnicalAnalysisRSI
from technical_analysis.so import TechnicalAnalysisSTOCH
from technical_analysis.williamsr import TechnicalAnalysisWilliamsR
from technical_analysis.wma import TechnicalAnalysisWMA
# option settings
pd.set_option("display.max_columns", 250)
pd.set_option("display.max_rows", 250)
import warnings
warnings.filterwarnings('ignore')
plt.rcParams['figure.figsize'] = (10.0, 20.0)
bitmex_exchange_client = ExchangeClient("bitmex", Path(config_ini))
mysql_client = DBClient("mysql", Path(config_ini))
dataset_manager = Dataset(mysql_client, bitmex_exchange_client)
dataset_manager.update_ohlcv("bitmex")
# manually added
def get_joined_params_and_summary():
bot = BottomTrendFollow(db_client=mysql_client,
exchange_client=bitmex_exchange_client,
is_backtest=True)
backtest_management = bot.backtest_management_table()
backtest_summary = BacktestSummary()
query_management = "SELECT * FROM " + bot.backtest_management_table_name + ";"
query_summary = "SELECT * FROM backtest_summary;"
backtest_management_df = mysql_client.exec_sql(query_management)
#!/usr/bin/python3
#
# Author: Steve Landherr <*****@*****.**>
#
# Script to extract standardized features from the
# comment submissions to FCC proceedings 17-108, driven by
# the Startup Policy Lab survey results.
#
# NOTE: Customize data extraction using the extract_spec.py module.
import sys
from lib.dataset import Dataset
def usage():
sys.stderr.write("usage: extract-dataset.py\n")
sys.exit(0)
if __name__ == '__main__':
if len(sys.argv) > 1:
usage()
sys.stderr.write('starting...\n')
dataset = Dataset()
dataset.pull_points_from_survey()
dataset.add_features_from_fcc()
dataset.filter_one_hot_features()
dataset.print_csv()
def main():
args = parse_args()
np.random.seed(args.seed)
cudnn.benchmark = False
cudnn.deterministic = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
if args.name is None:
args.name = '%s_%s' % (args.arch, datetime.now().strftime('%m%d%H'))
if not os.path.exists('models/%s' % args.name):
os.makedirs('models/%s' % args.name)
print('Config -----')
for arg in vars(args):
print('- %s: %s' % (arg, getattr(args, arg)))
print('------------')
with open('models/%s/args.txt' % args.name, 'w') as f:
for arg in vars(args):
print('- %s: %s' % (arg, getattr(args, arg)), file=f)
joblib.dump(args, 'models/%s/args.pkl' % args.name)
if args.loss == 'CrossEntropyLoss':
criterion = nn.CrossEntropyLoss().cuda()
elif args.loss == 'FocalLoss':
criterion = FocalLoss().cuda()
elif args.loss == 'MSELoss':
criterion = nn.MSELoss().cuda()
elif args.loss == 'multitask':
criterion = {
'classification': nn.CrossEntropyLoss().cuda(),
'regression': nn.MSELoss().cuda(),
}
else:
raise NotImplementedError
if args.pred_type == 'classification':
num_outputs = 5
elif args.pred_type == 'regression':
num_outputs = 1
elif args.loss == 'multitask':
num_outputs = 6
else:
raise NotImplementedError
train_transform = transforms.Compose([
transforms.Resize((args.img_size, args.img_size)),
transforms.RandomAffine(
degrees=(args.rotate_min, args.rotate_max) if args.rotate else 0,
translate=(args.translate_min,
args.translate_max) if args.translate else None,
scale=(args.rescale_min,
args.rescale_max) if args.rescale else None,
shear=(args.shear_min, args.shear_max) if args.shear else None,
),
transforms.CenterCrop(args.input_size),
transforms.RandomHorizontalFlip(p=0.5 if args.flip else 0),
transforms.RandomVerticalFlip(p=0.5 if args.flip else 0),
transforms.ColorJitter(brightness=0,
contrast=args.contrast,
saturation=0,
hue=0),
RandomErase(prob=args.random_erase_prob if args.random_erase else 0,
sl=args.random_erase_sl,
sh=args.random_erase_sh,
r=args.random_erase_r),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
])
val_transform = transforms.Compose([
transforms.Resize((args.img_size, args.input_size)),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
])
# data loading code
if 'diabetic_retinopathy' in args.train_dataset:
diabetic_retinopathy_dir = preprocess('diabetic_retinopathy',
args.img_size,
scale=args.scale_radius,
norm=args.normalize,
pad=args.padding,
remove=args.remove)
diabetic_retinopathy_df = pd.read_csv(
'inputs/diabetic-retinopathy-resized/trainLabels.csv')
diabetic_retinopathy_img_paths = \
diabetic_retinopathy_dir + '/' + diabetic_retinopathy_df['image'].values + '.jpeg'
diabetic_retinopathy_labels = diabetic_retinopathy_df['level'].values
if 'aptos2019' in args.train_dataset:
aptos2019_dir = preprocess('aptos2019',
args.img_size,
scale=args.scale_radius,
norm=args.normalize,
pad=args.padding,
remove=args.remove)
aptos2019_df = pd.read_csv('inputs/train.csv')
aptos2019_img_paths = aptos2019_dir + '/' + aptos2019_df[
'id_code'].values + '.png'
aptos2019_labels = aptos2019_df['diagnosis'].values
if 'chestxray' in args.train_dataset:
chestxray_dir = preprocess('chestxray',
args.img_size,
scale=args.scale_radius,
norm=args.normalize,
pad=args.padding,
remove=args.remove)
chestxray_img_paths = []
chestxray_labels = []
normal_cases = glob('chest_xray/chest_xray/train/NORMAL/*.jpeg')
pneumonia_cases = glob('chest_xray/chest_xray/train/PNEUMONIA/*.jpeg')
for nor in normal_cases:
p = nor.split('/')[-1]
chestxray_img_paths.append(chestxray_dir + '/' + p)
chestxray_labels.append(0)
for abn in pneumonia_cases:
p = abn.split('/')[-1]
chestxray_img_paths.append(chestxray_dir + '/' + p)
chestxray_labels.append(1)
normal_cases = glob('chest_xray/chest_xray/test/NORMAL/*.jpeg')
pneumonia_cases = glob('chest_xray/chest_xray/test/PNEUMONIA/*.jpeg')
for nor in normal_cases:
p = nor.split('/')[-1]
chestxray_img_paths.append(chestxray_dir + '/' + p)
chestxray_labels.append(0)
for abn in pneumonia_cases:
p = abn.split('/')[-1]
chestxray_img_paths.append(chestxray_dir + '/' + p)
chestxray_labels.append(1)
normal_cases = glob('chest_xray/chest_xray/val/NORMAL/*.jpeg')
pneumonia_cases = glob('chest_xray/chest_xray/val/PNEUMONIA/*.jpeg')
for nor in normal_cases:
p = nor.split('/')[-1]
chestxray_img_paths.append(chestxray_dir + '/' + p)
chestxray_labels.append(0)
for abn in pneumonia_cases:
p = abn.split('/')[-1]
chestxray_img_paths.append(chestxray_dir + '/' + p)
chestxray_labels.append(1)
chestxray_img_paths = np.array(chestxray_img_paths)
chestxray_labels = np.array(chestxray_labels)
if args.train_dataset == 'aptos2019':
skf = StratifiedKFold(n_splits=args.n_splits,
shuffle=True,
random_state=41)
img_paths = []
labels = []
for fold, (train_idx, val_idx) in enumerate(
skf.split(aptos2019_img_paths, aptos2019_labels)):
img_paths.append(
(aptos2019_img_paths[train_idx], aptos2019_img_paths[val_idx]))
labels.append(
(aptos2019_labels[train_idx], aptos2019_labels[val_idx]))
elif args.train_dataset == 'diabetic_retinopathy':
img_paths = [(diabetic_retinopathy_img_paths, aptos2019_img_paths)]
labels = [(diabetic_retinopathy_labels, aptos2019_labels)]
elif 'diabetic_retinopathy' in args.train_dataset and 'aptos2019' in args.train_dataset:
skf = StratifiedKFold(n_splits=args.n_splits,
shuffle=True,
random_state=41)
img_paths = []
labels = []
for fold, (train_idx, val_idx) in enumerate(
skf.split(aptos2019_img_paths, aptos2019_labels)):
img_paths.append((np.hstack((aptos2019_img_paths[train_idx],
diabetic_retinopathy_img_paths)),
aptos2019_img_paths[val_idx]))
labels.append((np.hstack(
(aptos2019_labels[train_idx], diabetic_retinopathy_labels)),
aptos2019_labels[val_idx]))
# FL setting: separate data into users
if 'diabetic_retinopathy' in args.train_dataset and 'aptos2019' in args.train_dataset:
combined_paths = np.hstack(
(aptos2019_img_paths, diabetic_retinopathy_img_paths))
combined_labels = np.hstack(
(aptos2019_labels, diabetic_retinopathy_labels))
elif 'chestxray' in args.train_dataset:
combined_paths = chestxray_img_paths
combined_labels = chestxray_labels
else:
raise NotImplementedError
user_ind_dict, ind_test = split_dataset(combined_labels, args.num_users,
args.iid)
model = get_model(model_name=args.arch,
num_outputs=num_outputs,
freeze_bn=args.freeze_bn,
dropout_p=args.dropout_p)
model = model.cuda()
test_set = Dataset(combined_paths[ind_test],
combined_labels[ind_test],
transform=val_transform)
test_loader = torch.utils.data.DataLoader(test_set,
batch_size=args.batch_size,
shuffle=False,
num_workers=4)
test_acc = []
test_scores = []
test_scores_f1 = []
lr = args.lr
for epoch in range(args.epochs):
print('Epoch [%d/%d]' % (epoch + 1, args.epochs))
weight_list = []
selected_ind = np.random.choice(args.num_users,
int(args.num_users / 10),
replace=False)
for i in selected_ind:
print('user: %d' % (i + 1))
train_set = Dataset(combined_paths[user_ind_dict[i]],
combined_labels[user_ind_dict[i]],
transform=train_transform)
train_loader = torch.utils.data.DataLoader(
train_set,
batch_size=args.batch_size,
shuffle=False if args.class_aware else True,
num_workers=4,
sampler=sampler if args.class_aware else None)
# train for one epoch
train_loss, train_score, ret_w = train(args, train_loader,
copy.deepcopy(model),
criterion, lr)
weight_list.append(ret_w)
print('loss %.4f - score %.4f' % (train_loss, train_score))
weights = fedavg(weight_list)
model.load_state_dict(weights)
test_loss, test_score, test_scoref1, accuracy, confusion_matrix = test(
args, test_loader, copy.deepcopy(model), criterion)
print('loss %.4f - score %.4f - accuracy %.4f' %
(test_loss, test_score, accuracy))
test_acc.append(accuracy)
test_scores.append(test_score)
test_scores_f1.append(test_scoref1)
lr *= 0.992
np.savez('./accuracy-xray-iid' + str(args.iid) + '-' + str(args.epochs) +
'-beta' + str(args.beta) + '-seed' + str(args.seed),
acc=np.array(test_acc),
score=np.array(test_scores),
scoref1=np.array(test_scores_f1),
confusion=confusion_matrix)
def main():
args = parse_args()
if args.name is None:
args.name = '%s_%s' % (args.arch, datetime.now().strftime('%m%d%H'))
if not os.path.exists('models/%s' % args.name):
os.makedirs('models/%s' % args.name)
if args.resume:
args = joblib.load('models/%s/args.pkl' % args.name)
args.resume = True
print('Config -----')
for arg in vars(args):
print('- %s: %s' % (arg, getattr(args, arg)))
print('------------')
with open('models/%s/args.txt' % args.name, 'w') as f:
for arg in vars(args):
print('- %s: %s' % (arg, getattr(args, arg)), file=f)
joblib.dump(args, 'models/%s/args.pkl' % args.name)
if args.seed is not None and not args.resume:
print('set random seed')
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if args.loss == 'BCEWithLogitsLoss':
criterion = BCEWithLogitsLoss().cuda()
elif args.loss == 'WeightedBCEWithLogitsLoss':
criterion = BCEWithLogitsLoss(weight=torch.Tensor([1., 1., 1., 1., 1., 2.]),
smooth=args.label_smooth).cuda()
elif args.loss == 'FocalLoss':
criterion = FocalLoss().cuda()
elif args.loss == 'WeightedFocalLoss':
criterion = FocalLoss(weight=torch.Tensor([1., 1., 1., 1., 1., 2.])).cuda()
else:
raise NotImplementedError
if args.pred_type == 'all':
num_outputs = 6
elif args.pred_type == 'except_any':
num_outputs = 5
else:
raise NotImplementedError
cudnn.benchmark = True
# create model
model = get_model(model_name=args.arch,
num_outputs=num_outputs,
freeze_bn=args.freeze_bn,
dropout_p=args.dropout_p,
pooling=args.pooling,
lp_p=args.lp_p)
model = model.cuda()
train_transform = Compose([
transforms.Resize(args.img_size, args.img_size),
transforms.HorizontalFlip() if args.hflip else NoOp(),
transforms.VerticalFlip() if args.vflip else NoOp(),
transforms.ShiftScaleRotate(
shift_limit=args.shift_limit,
scale_limit=args.scale_limit,
rotate_limit=args.rotate_limit,
border_mode=cv2.BORDER_CONSTANT,
value=0,
p=args.shift_scale_rotate_p
) if args.shift_scale_rotate else NoOp(),
transforms.RandomContrast(
limit=args.contrast_limit,
p=args.contrast_p
) if args.contrast else NoOp(),
RandomErase() if args.random_erase else NoOp(),
transforms.CenterCrop(args.crop_size, args.crop_size) if args.center_crop else NoOp(),
ForegroundCenterCrop(args.crop_size) if args.foreground_center_crop else NoOp(),
transforms.RandomCrop(args.crop_size, args.crop_size) if args.random_crop else NoOp(),
transforms.Normalize(mean=model.mean, std=model.std),
ToTensor(),
])
if args.img_type:
stage_1_train_dir = 'processed/stage_1_train_%s' %args.img_type
else:
stage_1_train_dir = 'processed/stage_1_train'
df = pd.read_csv('inputs/stage_1_train.csv')
img_paths = np.array([stage_1_train_dir + '/' + '_'.join(s.split('_')[:-1]) + '.png' for s in df['ID']][::6])
labels = np.array([df.loc[c::6, 'Label'].values for c in range(6)]).T.astype('float32')
df = df[::6]
df['img_path'] = img_paths
for c in range(6):
df['label_%d' %c] = labels[:, c]
df['ID'] = df['ID'].apply(lambda s: '_'.join(s.split('_')[:-1]))
meta_df = pd.read_csv('processed/stage_1_train_meta.csv')
meta_df['ID'] = meta_df['SOPInstanceUID']
test_meta_df = pd.read_csv('processed/stage_1_test_meta.csv')
df = pd.merge(df, meta_df, how='left')
patient_ids = meta_df['PatientID'].unique()
test_patient_ids = test_meta_df['PatientID'].unique()
if args.remove_test_patient_ids:
patient_ids = np.array([s for s in patient_ids if not s in test_patient_ids])
train_img_paths = np.hstack(df[['img_path', 'PatientID']].groupby(['PatientID'])['img_path'].apply(np.array).loc[patient_ids].to_list()).astype('str')
train_labels = []
for c in range(6):
train_labels.append(np.hstack(df[['label_%d' %c, 'PatientID']].groupby(['PatientID'])['label_%d' %c].apply(np.array).loc[patient_ids].to_list()))
train_labels = np.array(train_labels).T
if args.resume:
checkpoint = torch.load('models/%s/checkpoint.pth.tar' % args.name)
# train
train_set = Dataset(
train_img_paths,
train_labels,
transform=train_transform)
train_loader = torch.utils.data.DataLoader(
train_set,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.num_workers,
# pin_memory=True,
)
if args.optimizer == 'Adam':
optimizer = optim.Adam(
filter(lambda p: p.requires_grad, model.parameters()), lr=args.lr, weight_decay=args.weight_decay)
elif args.optimizer == 'AdamW':
optimizer = optim.AdamW(
filter(lambda p: p.requires_grad, model.parameters()), lr=args.lr, weight_decay=args.weight_decay)
elif args.optimizer == 'RAdam':
optimizer = RAdam(
filter(lambda p: p.requires_grad, model.parameters()), lr=args.lr, weight_decay=args.weight_decay)
elif args.optimizer == 'SGD':
optimizer = optim.SGD(filter(lambda p: p.requires_grad, model.parameters()), lr=args.lr,
momentum=args.momentum, weight_decay=args.weight_decay, nesterov=args.nesterov)
else:
raise NotImplementedError
if args.apex:
amp.initialize(model, optimizer, opt_level='O1')
if args.scheduler == 'CosineAnnealingLR':
scheduler = lr_scheduler.CosineAnnealingLR(
optimizer, T_max=args.epochs, eta_min=args.min_lr)
elif args.scheduler == 'MultiStepLR':
scheduler = lr_scheduler.MultiStepLR(optimizer,
milestones=[int(e) for e in args.milestones.split(',')], gamma=args.gamma)
else:
raise NotImplementedError
log = {
'epoch': [],
'loss': [],
}
start_epoch = 0
if args.resume:
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
scheduler.load_state_dict(checkpoint['scheduler'])
start_epoch = checkpoint['epoch']
log = pd.read_csv('models/%s/log.csv' % args.name).to_dict(orient='list')
for epoch in range(start_epoch, args.epochs):
print('Epoch [%d/%d]' % (epoch + 1, args.epochs))
# train for one epoch
train_loss = train(args, train_loader, model, criterion, optimizer, epoch)
if args.scheduler == 'CosineAnnealingLR':
scheduler.step()
print('loss %.4f' % (train_loss))
log['epoch'].append(epoch)
log['loss'].append(train_loss)
pd.DataFrame(log).to_csv('models/%s/log.csv' % args.name, index=False)
torch.save(model.state_dict(), 'models/%s/model.pth' % args.name)
print("=> saved model")
state = {
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict(),
'scheduler': scheduler.state_dict(),
}
torch.save(state, 'models/%s/checkpoint.pth.tar' % args.name)
def main():
parser = argparse.ArgumentParser(
description=
"Generate some useful quantile graphs and perhaps a few histograms too"
)
parser.add_argument("data_directory",
help="the location where we are storing the data.")
args = parser.parse_args()
not_enough_data = []
def perform_analysis(az, insttype, data_before, data_after):
data_before = PriceTimeSeriesNaive(data_before)
data_after = PriceTimeSeriesNaive(data_after)
days_required = 90
bucket_interval = 350 # bucket interval in seconds
bucket_before = PriceBucketNaive(
data_before.expand_with_interval(bucket_interval))
bucket_after = PriceBucketNaive(
data_after.expand_with_interval(bucket_interval))
if len(bucket_after.by_ts) == 0 or len(bucket_before.by_ts) == 0:
return
SECONDS_IN_DAY = 3600 * 24
SIZE_REQUIREMENT = SECONDS_IN_DAY / bucket_interval * days_required
if bucket_before.size() < SECONDS_IN_DAY or bucket_after.size(
) < SECONDS_IN_DAY:
print("\tnot enough data for region")
bucket_before = bucket_before.shrink_to_size(min_size)
bucket_after = bucket_after.shrink_to_size(min_size)
mean = bucket_before.getAverage()
median = bucket_before.getQuantile(0.5)["price"]
print("\tprice window duration: %f" %
((bucket_before.by_ts[-1]["timestamp"] -
bucket_before.by_ts[0]["timestamp"]) / (3600 * 24)))
print("\tmean before/after: %f/%f" % (float(
bucket_before.getAverage()), float(bucket_after.getAverage())))
print("\tmedian before/after: %f/%f" %
(float(bucket_after.getQuantile(0.5)["price"]),
bucket_after.getQuantile(0.5)["price"]))
total_cost_before = sum(r["price"] for r in bucket_before.by_ts)
print("\ttotal/avg cost before: %f/%f" %
(total_cost_before,
total_cost_before / float(len(bucket_before.by_ts))))
total_cost_after = sum(r["price"] for r in bucket_after.by_ts)
print("\ttotal/avg cost after: %f/%f" %
(total_cost_after,
total_cost_after / float(len(bucket_before.by_ts))))
print("\ttotal/avg price difference: %f/%f" %
(total_cost_after - total_cost_before,
(total_cost_after - total_cost_before) /
float(len(bucket_before.by_ts))))
# return the result object containing aggregates of the values we computed
return {
"key": (az, insttype),
"'popularity'": len(data_before.data) + len(data_after.data),
"total_cost_before": total_cost_before,
"avg_cost_before":
total_cost_before / float(len(bucket_before.by_ts)),
"total_cost_after": total_cost_after,
"avg_cost_after":
total_cost_after / float(len(bucket_before.by_ts))
}
def transform_query_results(results):
return ({
"timestamp": result[0],
"price": result[1]
} for result in results)
results = []
with Dataset(args.data_directory) as dataset:
count = 0
for az, insttype in dataset.get_databases(
): # misleading function name
conn = dataset.open(az, insttype)
# compute the most recent timestamp so we can determine a good
# duration for the intervals we will be examining
c = conn.cursor()
c.execute("SELECT MAX(timestamp) AS timestamp FROM prices")
most_recent_ts = c.fetchone()[0]
interval_duration = most_recent_ts - AFTER_START_EPOCH
if interval_duration < 0: continue
# fetch the data for the before window
c = conn.cursor()
c.execute(
"SELECT timestamp, price FROM prices "
" WHERE timestamp > %d AND timestamp < %d ORDER BY timestamp" %
(BEFORE_END_EPOCH - interval_duration, BEFORE_END_EPOCH))
data_before = list(transform_query_results(c.fetchall()))
c = conn.cursor()
c.execute("SELECT timestamp, price FROM prices " +
" WHERE timestamp > %d ORDER BY timestamp " %
(AFTER_START_EPOCH))
data_after = list(transform_query_results(c.fetchall()))
if len(data_before) == 0 or len(data_after) == 0: continue
print("analyzing data for %s - %s" % (az, insttype))
results.append(
perform_analysis(az, insttype, data_before, data_after))
# count += 1
# if count > 100:
# break
results = list(filter(lambda x: x is not None, results))
results.sort(key=lambda r: r["'popularity'"])
print("Information filtered by instance popularity")
def aggregate_dictionary(dicts, aggfunc):
return {
key: aggfunc([d[key] for d in dicts])
for key in sorted(dicts[0].keys())
if type(dicts[0][key]) == int or type(dicts[0][key]) == float
}
def aggmedian(values):
return sorted(values)[len(values) // 2]
def aggaverage(values):
return sum(values) / float(len(values))
def print_results(popdict, unpopdict):
for key in popdict:
print("\t%s popular/unpopular: %f/%f" %
(key, popdict[key], unpopdict[key]))
popular_inst_types = [
"m1.small", "m1.large", "t1.micro", "m1.medium", "c1.xlarge",
"c1.medium"
]
results_popular = list(
filter(lambda x: x["key"][1] in popular_inst_types, results))
results_unpopular = list(
filter(lambda x: x["key"][1] not in popular_inst_types, results))
print("results (median, hard coded popular instance types)")
print("%d popular results found" % (len(results_popular)))
pop_results = aggregate_dictionary(results_popular, aggaverage)
unpop_results = aggregate_dictionary(results_unpopular, aggaverage)
print_results(pop_results, unpop_results)
