def __get_db_paths(self):
output_dir = get_output_dir(client_file=False)
self.db_path = os.path.join(output_dir, DATABASE_PATH)
self.db_backup_b4_path = os.path.join(output_dir,
BACKUP_DB_BEFORE_NAME)
self.db_backup_after_path = os.path.join(output_dir,
BACKUP_DB_AFTER_NAME)
def main():
# Get dataset
# Will be downloaded on the first run
train_images, train_labels, valid_images, valid_labels = data.get_fashion_mnist(
)
# Show some images from the dataset
data.show_random_images(train_images, train_labels, 50)
# Directory to save summaries to
# From your conda environment run
# tensorbard --logdir ../tf_playground/output
# to see training details
output = utils.get_output_dir()
# Create model
# TODO: Go to model.py file to implement model
model = Model()
# Lets train
# TODO: Go to trainer.py file to implement trainer
trainer = Trainer(train_images=train_images,
train_labels=train_labels,
valid_images=valid_images,
valid_labels=valid_labels,
model=model,
epochs=10,
batch_size=10,
output=output)
trainer.train()
trainer.save_final_accuracy()
def main():
# Generate and split data
# Try and play with arguments
all_data = data.generate_data_gauss(numSamples=1000, noise=0.5)
train_data, valid_data = data.split_data(all_data, val_factor=0.3)
# Set show to True if you want to see generated dataset
data.plot_data(train_data, valid_data, show=False)
# Directory to save summaries to
# From your conda environment run
# tensorbard --logdir ../tf_playground/output
# to see training details
output = utils.get_output_dir()
# Create model
# Go to model.py file to make changes to the model
model = Model()
# Lets train
# Try changing number of epochs and batch_size
trainer = Trainer(train_data=train_data,
valid_data=valid_data,
model=model,
epochs=10,
batch_size=2,
output=output)
trainer.train()
trainer.save_final_accuracy()
def copy_ui():
"""
Copy UI files (index.html and handlebars.js) from the linkbak/ui/ path
to the output directory
"""
output_dir = get_output_dir()
readpath = __loader__.path[:__loader__.path.rfind('/')]
for uifile in ["index.html", "handlebars.js"]:
target_path = Path(f"{output_dir}/{uifile}")
if target_path.exists():
target_path.unlink()
copyfile(f"{readpath}/ui/{uifile}", target_path)
get_logger().warning("cd output && python -m http.server")
def merge_json():
"""
Merge all JSON files
"""
results = []
output_dir = get_output_dir()
with open(f'{output_dir}/results.json', 'w') as json_results_fp:
with flocked(json_results_fp):
for jsonfile in Path(output_dir).glob("*/metadata.json"):
if str(jsonfile) == f"{output_dir}/results.json":
continue
get_logger().debug("Appending %s", str(jsonfile))
with jsonfile.open() as json_fp:
results.append(json.load(json_fp))
json.dump(results, json_results_fp, indent=True)
def main():
"""
Run the script
"""
args = parse_args()
setup_logging(args)
outdir = Path(get_output_dir())
if not outdir.exists():
outdir.mkdir()
copy_ui()
nb_workers = args.j if args.j else os.cpu_count()
get_logger().warning("Using %s workers", nb_workers)
if nb_workers > 1:
with contextlib.closing(multiprocessing.Pool(nb_workers)) as pool:
pool.starmap(start_link_handler,
[(l, args) for l in get_links(args.file[0])])
else:
for link in get_links(args.file[0]):
start_link_handler(link, args)
def main() -> None:
args = parser.parse_args()
#Load model
depth_model_dir = get_output_dir(args)
print("args.output_dir =", args.output_dir)
print("depth_model_dir =", depth_model_dir)
depth_model_cp_fn = os.path.join(depth_model_dir, "model_best.pth.tar")
depth_model_cp = torch.load(depth_model_cp_fn)
rgb_model_cp = torch.load(rgbd_model_locations[args.rgb_model_kind])
args.data = os.path.join( #type:ignore
os.environ["DATASET_DIR"], args.data)
depth_model: Type[ResNet] = depth_model_cp["model"]
rgb_model: Type[ResNet] = rgb_model_cp["model"]
# load dataset
valdir = os.path.join(args.data, 'val')
image_shape = (192, 256)
val_dataset_rgb = NYUDataset(valdir,
phase='val',
modality="rgb",
num_samples=0,
square_width=args.square_width,
output_shape=image_shape)
val_dataloader_rgb = torch.utils.data.DataLoader(val_dataset_rgb)
val_dataset_rgbd = NYUDataset(valdir,
phase='val',
modality="rgbd",
num_samples=0,
square_width=args.square_width,
output_shape=image_shape,
depth_type=args.depth_type)
val_dataloader_rgbd = torch.utils.data.DataLoader(val_dataset_rgbd)
depth_model_is_rgbd = depth_model.in_channels == 4
rgb_model_is_rgbd = rgb_model.in_channels == 4
depth_results = evaluate(
depth_model,
val_dataloader_rgbd if depth_model_is_rgbd else val_dataloader_rgb)
rgb_results = evaluate(
rgb_model,
val_dataloader_rgbd if rgb_model_is_rgbd else val_dataloader_rgb)
sort_metric_depth = np.array([(result.result_outside if depth_model_is_rgbd
else result.result).absrel
for result in depth_results])
sort_metric_rgb = np.array([
(result.result_outside if rgb_model_is_rgbd else result.result).absrel
for result in rgb_results
])
ratio = sort_metric_depth / sort_metric_rgb
acending_ratio_idxs = np.argsort(ratio)
decending_ratio_idxs = np.flip(acending_ratio_idxs, 0)
print(f"=> number of images where ratio > 1: {np.sum(ratio > 1)}")
print(f"=> total number of images: {sort_metric_depth.size}")
print(
f"=> ration of images where ratio > 1: {np.sum(ratio > 1)/sort_metric_depth.size}"
)
#plot error distribution
fig, ax1 = plt.subplots()
ax1.plot(ratio[acending_ratio_idxs], "y-", zorder=0)
ax1.set_ylabel(
f"$\\frac{{absrel_{{{args.depth_name}}} }}{{absrel_{{{args.rgb_name}}} }}$"
)
ax2 = ax1.twinx()
ax2.set_ylabel("absrel")
ax2.plot(sort_metric_depth[acending_ratio_idxs])
ax2.plot(sort_metric_rgb[acending_ratio_idxs], "--")
plt.title("Difference in error")
ax1.legend([
f"$\\frac{{absrel_{{{args.depth_name}}} }}{{absrel_{{{args.rgb_name}}} }}$"
])
ax2.legend([
f"$absrel_{{{args.depth_name}}}$",
f"$absrel_{{{args.rgb_name}}}$",
])
ax1.set_xlabel("image")
plt.savefig(os.path.join(depth_model_dir, "difference_in_error.png"))
#plot images from high difference in error to low
def draw_images(idxs: Iterable[int]) -> np.ndarray:
im_merge = None
for i in idxs:
plt.figure()
x_rgbd, y = val_dataset_rgbd[i]
x_rgb, y = val_dataset_rgb[i]
x_rgb = torch.unsqueeze(x_rgb, 0)
x_rgbd = torch.unsqueeze(x_rgbd, 0)
y = torch.unsqueeze(y, 0)
x_rgbd, x_rgb, y = x_rgbd.cuda(), x_rgb.cuda(), y.cuda()
y_hat_depth = depth_model(x_rgbd # type: ignore
if depth_model_is_rgbd else x_rgb)
y_hat_rgb = rgb_model(x_rgbd # type: ignore
if rgb_model_is_rgbd else x_rgb)
images = [x_rgbd, y, y_hat_depth, y_hat_rgb]
squares = None if "single-pixel" in args.depth_type else [
val_dataset_rgbd.square
] * len(images)
row = merge_ims_into_row(images, square=squares)
im_merge = row if im_merge is None else add_row(im_merge, row)
return im_merge
depth_model_best_im = draw_images(acending_ratio_idxs[:args.imrows])
print("depth_model_best_im ratios =",
ratio[acending_ratio_idxs[:args.imrows]])
depth_model_worst_im = draw_images(decending_ratio_idxs[:args.imrows])
print("depth_model_worst_im ratios =",
ratio[decending_ratio_idxs[:args.imrows]])
save_image(depth_model_best_im,
os.path.join(depth_model_dir, f"{args.depth_name}_best.png"))
save_image(depth_model_worst_im,
os.path.join(depth_model_dir, f"{args.depth_name}_worst.png"))
def main() -> int:
best_result = Result()
best_result.set_to_worst()
args: Any
args = parser.parse_args()
dataset = args.data
if args.modality == 'rgb' and args.num_samples != 0:
print("number of samples is forced to be 0 when input modality is rgb")
args.num_samples = 0
image_shape = (192, 256) # if "my" in args.arch else (228, 304)
# create results folder, if not already exists
if args.transfer_from:
output_directory = f"{args.transfer_from}_transfer"
else:
output_directory = utils.get_output_dir(args)
args.data = os.path.join(os.environ["DATASET_DIR"], args.data)
print("output directory :", output_directory)
if not os.path.exists(output_directory):
os.makedirs(output_directory)
elif not args.evaluate:
raise Exception("output directory allready exists")
train_csv = os.path.join(output_directory, 'train.csv')
test_csv = os.path.join(output_directory, 'test.csv')
best_txt = os.path.join(output_directory, 'best.txt')
# define loss function (criterion) and optimizer
if args.criterion == 'l2':
criterion = criteria.MaskedMSELoss().cuda()
elif args.criterion == 'l1':
criterion = criteria.MaskedL1Loss().cuda()
out_channels = 1
# Data loading code
print("=> creating data loaders ...")
traindir = os.path.join(args.data, 'train')
valdir = traindir if dataset == "SUNRGBD" else os.path.join(
args.data, 'val')
DatasetType = choose_dataset_type(dataset)
train_dataset = DatasetType(traindir,
phase='train',
modality=args.modality,
num_samples=args.num_samples,
square_width=args.square_width,
output_shape=image_shape,
depth_type=args.depth_type)
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True,
sampler=None)
print("=> training examples:", len(train_dataset))
val_dataset = DatasetType(valdir,
phase='val',
modality=args.modality,
num_samples=args.num_samples,
square_width=args.square_width,
output_shape=image_shape,
depth_type=args.depth_type)
val_loader = torch.utils.data.DataLoader(val_dataset,
batch_size=1,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
print("=> validation examples:", len(val_dataset))
print("=> data loaders created.")
# evaluation mode
if args.evaluate:
best_model_filename = os.path.join(output_directory,
'model_best.pth.tar')
if os.path.isfile(best_model_filename):
print("=> loading best model '{}'".format(best_model_filename))
checkpoint = torch.load(best_model_filename)
args.start_epoch = checkpoint['epoch']
best_result = checkpoint['best_result']
model = checkpoint['model']
print("=> loaded best model (epoch {})".format(
checkpoint['epoch']))
else:
print("=> no best model found at '{}'".format(best_model_filename))
avg_result, avg_result_inside, avg_result_outside, _, results, evaluator = validate(
val_loader,
args.square_width,
args.modality,
output_directory,
args.print_freq,
test_csv,
model,
checkpoint['epoch'],
write_to_file=False)
write_results(best_txt, avg_result, avg_result_inside,
avg_result_outside, checkpoint['epoch'])
for loss_name, losses in [
("rmses", (res.result.rmse for res in results)),
("delta1s", (res.result.delta1 for res in results)),
("delta2s", (res.result.delta2 for res in results)),
("delta3s", (res.result.delta3 for res in results)),
("maes", (res.result.mae for res in results)),
("absrels", (res.result.absrel for res in results)),
("rmses_inside", (res.result_inside.rmse for res in results)),
("delta1s_inside", (res.result_inside.delta1 for res in results)),
("delta2s_inside", (res.result_inside.delta2 for res in results)),
("delta3s_inside", (res.result_inside.delta3 for res in results)),
("maes_inside", (res.result_inside.mae for res in results)),
("absrels_inside", (res.result_inside.absrel for res in results)),
("rmses_outside", (res.result_outside.rmse for res in results)),
("delta1s_outside", (res.result_outside.delta1
for res in results)),
("delta2s_outside", (res.result_outside.delta2
for res in results)),
("delta3s_outside", (res.result_outside.delta3
for res in results)),
("maes_outside", (res.result_outside.mae for res in results)),
("absrels_outside", (res.result_outside.absrel
for res in results)),
]:
with open(
os.path.join(output_directory,
f"validation_{loss_name}.csv"),
"w") as csv_file:
wr = csv.writer(csv_file, quoting=csv.QUOTE_ALL)
wr.writerow(losses)
evaluator.save_plot(os.path.join(output_directory, "best.png"))
return 0
# optionally resume from a checkpoint
elif args.resume:
if os.path.isfile(args.resume):
print("=> loading checkpoint '{}'".format(args.resume))
checkpoint = torch.load(args.resume)
args.start_epoch = checkpoint['epoch'] + 1
best_result = checkpoint['best_result']
model = checkpoint['model']
optimizer = checkpoint['optimizer']
print("=> loaded checkpoint (epoch {})".format(
checkpoint['epoch']))
else:
print("=> no checkpoint found at '{}'".format(args.resume))
return 1
# create new model
else:
if args.transfer_from:
if os.path.isfile(args.transfer_from):
print(f"=> loading checkpoint '{args.transfer_from}'")
checkpoint = torch.load(args.transfer_from)
args.start_epoch = 0
model = checkpoint['model']
print("=> loaded checkpoint")
train_params = list(model.conv3.parameters()) + list(
model.decoder.layer4.parameters(
)) if args.train_top_only else model.parameters()
else:
print(f"=> no checkpoint found at '{args.transfer_from}'")
return 1
else:
# define model
print("=> creating Model ({}-{}) ...".format(
args.arch, args.decoder))
in_channels = len(args.modality)
if args.arch == 'resnet50':
n_layers = 50
elif args.arch == 'resnet18':
n_layers = 18
model = ResNet(layers=n_layers,
decoder=args.decoder,
in_channels=in_channels,
out_channels=out_channels,
pretrained=args.pretrained,
image_shape=image_shape,
skip_type=args.skip_type)
print("=> model created.")
train_params = model.parameters()
adjusting_learning_rate = False
if args.optimizer == "sgd":
optimizer = torch.optim.SGD(train_params,
args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
adjusting_learning_rate = True
elif args.optimizer == "adam":
optimizer = torch.optim.Adam(train_params,
weight_decay=args.weight_decay)
else:
raise Exception("We should never be here")
if adjusting_learning_rate:
print("=> Learning rate adjustment enabled.")
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
optimizer, patience=args.adjust_lr_ep, verbose=True)
# create new csv files with only header
with open(train_csv, 'w') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
with open(test_csv, 'w') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
# model = torch.nn.DataParallel(model).cuda()
model = model.cuda()
print(model)
print("=> model transferred to GPU.")
epochs_since_best = 0
train_results = []
val_results = []
for epoch in range(args.start_epoch, args.epochs):
# train for one epoch
res_train, res_train_inside, res_train_outside = train(
train_loader, model, criterion, optimizer, epoch, args.print_freq,
train_csv)
train_results.append((res_train, res_train_inside, res_train_outside))
# evaluate on validation set
res_val, res_val_inside, res_val_outside, img_merge, _, _ = validate(
val_loader, args.square_width, args.modality, output_directory,
args.print_freq, test_csv, model, epoch, True)
val_results.append((res_val, res_val_inside, res_val_outside))
# remember best rmse and save checkpoint
is_best = res_val.rmse < best_result.rmse
if is_best:
epochs_since_best = 0
best_result = res_val
write_results(best_txt, res_val, res_val_inside, res_val_outside,
epoch)
if img_merge is not None:
img_filename = output_directory + '/comparison_best.png'
utils.save_image(img_merge, img_filename)
else:
epochs_since_best += 1
save_checkpoint(
{
'epoch': epoch,
'arch': args.arch,
'model': model,
'best_result': best_result,
'optimizer': optimizer,
}, is_best, epoch, output_directory)
plot_progress(train_results, val_results, epoch, output_directory)
if epochs_since_best > args.early_stop_epochs:
print("early stopping")
if adjusting_learning_rate:
scheduler.step(res_val.rmse)
return 0
def replace_old_testing_json(raw_orders, json_fname: str):
'''deletes old json, exports raw orders to json file'''
output_dir = get_output_dir(client_file=False)
json_path = os.path.join(output_dir, json_fname)
delete_file(json_path)
dump_to_json(raw_orders, json_fname)
},
]
# GLOBAL VARIABLES
TESTING = False
TEST_CASE = TEST_CASES[2]
# TEST_TODAY_DATE = '2022-06-27'
TEST_TODAY_DATE = datetime.now().strftime(
'%Y-%m-%d') # Hardcode in format: '2021-08-19' if needed when testing
SALES_CHANNEL = TEST_CASE['channel']
ORDERS_SOURCE_FILE = TEST_CASE['file']
EXPECTED_SYS_ARGS = 3
# Logging config:
log_path = os.path.join(get_output_dir(client_file=False), 'report.log')
logging.basicConfig(handlers=[logging.FileHandler(log_path, 'a', 'utf-8')],
level=logging.INFO)
def get_cleaned_orders(source_file: str, sales_channel: str,
proxy_keys: dict) -> list:
'''returns cleaned orders (as cleaned in clean_orders func) from source_file arg path'''
encoding, delimiter = get_file_encoding_delimiter(source_file)
logging.info(
f'{os.path.basename(source_file)} detected encoding: {encoding}, delimiter <{delimiter}>'
)
raw_orders = get_raw_orders(source_file, encoding, delimiter)
logging.info(
f'Loaded {os.path.basename(source_file)} has {len(raw_orders)} raw orders. Filtering out todays orders...'
)
def _get_EU_countries_list_from_file(self):
'''returns list of EU member countries from TXT file'''
current_dir = get_output_dir(client_file=False)
txt_abspath = os.path.join(current_dir, EU_COUNTRIES_TXT)
logging.debug(f'Trying to access EU countries txt file: {txt_abspath}')
return get_EU_countries_from_txt(txt_abspath)
def _prepare_filepaths(self):
'''creates cls variables of files abs paths to be created one dir above this script dir'''
output_dir = get_output_dir()
date_stamp = datetime.today().strftime("%Y.%m.%d %H.%M")
self.report_path = os.path.join(
output_dir, f'{self.sales_channel} Report {date_stamp}.xlsx')