def generate_readmes(self, output_dir: str, paths: Dict[str, str]):
"""
Generate README output. We will also handle README context in memory
Args:
output_dir (:obj:`str`):
target directory
paths (:obj:`Dict[str, str]`):
Dictionary of release tags and target dockerfile.
"""
# this will include both yatai-service and models-server path.
_release_context: MutableMapping = defaultdict(list)
# check if given distro is supported per package.
for distro_version in self.releases.keys():
if 'debian' in distro_version:
_os_tag = 'slim'
elif 'amazonlinux' in distro_version:
_os_tag = 'ami'
else:
_os_tag = distro_version
_release_context[distro_version] = sorted(
[(release_tags, gen_path)
for release_tags, gen_path in paths.items()
if _os_tag in release_tags and 'base' not in release_tags],
key=operator.itemgetter(0),
)
_readme_context: Dict = {
'ephemeral': False,
'bentoml_package': "",
'bentoml_release_version': FLAGS.bentoml_version,
'release_info': _release_context,
}
for package in self.packages.keys():
output_readme = Path('docs', package)
shutil.rmtree(package, ignore_errors=True)
set_data(_readme_context, package, 'bentoml_package')
_readme_context['oss'] = maxkeys(self.packages[package])
self.render(
input_path=README_TEMPLATE,
output_path=output_readme,
metadata=_readme_context,
)
# renders README for generated directory
_readme_context['ephemeral'] = True
if not Path(output_dir, "README.md").exists():
self.render(
input_path=README_TEMPLATE,
output_path=Path(output_dir),
metadata=_readme_context,
)
def generate_readmes(self, output_dir: str, paths: t.Dict[str,
str]) -> None:
"""
Generate README output. We will also handle README context in memory
Args:
output_dir (:obj:`str`):
target directory
paths (:obj:`t.Dict[str, str]`):
t.Dictionary of release tags and target dockerfile.
"""
_release_context: t.MutableMapping = defaultdict(list)
# check if given distro is supported per package.
distros = [
self.releases[release]["add_to_tags"]
for release in self.releases.keys()
]
for distro_version in distros:
_release_context[distro_version] = sorted(
[(release_tags, gen_path)
for release_tags, gen_path in paths.items()
if distro_version in release_tags
and "base" not in release_tags],
key=operator.itemgetter(0),
)
_readme_context: t.Dict = {
"ephemeral": False,
"bentoml_package": "",
"bentoml_release_version": FLAGS.bentoml_version,
"release_info": _release_context,
}
for package in self.packages.keys():
output_readme = Path("generated", package)
set_data(_readme_context, package, "bentoml_package")
self.render(
input_path=README_TEMPLATE,
output_path=output_readme,
metadata=_readme_context,
)
# renders README for generated directory
_readme_context["ephemeral"] = True
if not Path(output_dir, "README.md").exists():
self.render(
input_path=README_TEMPLATE,
output_path=Path(output_dir),
metadata=_readme_context,
)
def dockerfiles(self, pkg: str, distro: str, pyv: str) -> Dict:
"""
Generate template context for each distro releases.
Args:
pkg: release bentoml packages, either model-server or yatai-service
distro: linux distro. eg: centos
pyv: python version
Returns:
build context that can be used with self.render
"""
_cuda_comp: Dict = get_data(self.cuda,
self._cuda_version).copy() # fmt: skip
# TODO: Better type annotation for nested dict.
_ctx: Dict = self.releases[distro].copy()
_ctx['package'] = pkg
# setup envars
_ctx['envars'] = self.envars()(_ctx['envars'])
# set PYTHON_VERSION envars.
set_data(_ctx, pyv, 'envars', 'PYTHON_VERSION') # fmt: skip
# setup cuda deps
if _ctx["cuda_prefix_url"]:
major, minor, _ = self._cuda_version.rsplit(".")
_ctx["cuda"] = {
"requires": _ctx['cuda_requires'],
"ml_repo": NVIDIA_ML_REPO_URL.format(_ctx["cuda_prefix_url"]),
"base_repo": NVIDIA_REPO_URL.format(_ctx["cuda_prefix_url"]),
"version": {
"major": major,
"minor": minor,
"shortened": f"{major}.{minor}",
},
"components": _cuda_comp,
"cudnn": self.cudnn(_cuda_comp),
}
_ctx.pop('cuda_prefix_url')
_ctx.pop('cuda_requires')
else:
log.debug(f"CUDA is disabled for {distro}. Skipping...")
for key in _ctx.keys():
if DOCKERFILE_NVIDIA_REGEX.match(key):
_ctx.pop(key)
return _ctx
def save_supermarket():
file = request.files['image']
supermkt = {
"id": str(uuid.uuid1()),
"name": request.form.get('name'),
"location": request.form.get('location'),
"img_name": file.filename
}
if get_data(LIST_SUPERMARKET):
data = get_data(LIST_SUPERMARKET)
data.append(supermkt)
set_data(data, LIST_SUPERMARKET)
else:
set_data([supermkt], LIST_SUPERMARKET)
if file:
path = os.path.join(UPLOAD_FOLDER, file.filename)
file.save(path)
return redirect(url_for('supermarkets.get_supermarkets'))
def save_product():
file = request.files['image']
prod = {
"id": str(uuid.uuid1()),
"name": request.form.get('name'),
"description": request.form.get('description'),
"img_name": file.filename,
"price": request.form.get('price')
}
if get_data(LIST_PRODUCT):
data = get_data(LIST_PRODUCT)
data.append(prod)
set_data(data, LIST_PRODUCT)
else:
set_data([prod], LIST_PRODUCT)
if file:
path = os.path.join(UPLOAD_FOLDER, file.filename)
file.save(path)
return redirect(url_for('products.get_products'))
def set_data_for_game(data, fields_game=[], fields_all=[], fields_self=[]):
if fields_game:
set_data(data, GAME_STATUS, fields_game)
if fields_all:
for player in data["players"]:
playerName = player["playerName"]
player_local = GAME_STATUS["players"][playerName]
set_data(player, player_local, fields_all)
if fields_self:
player_self = data["self"]
player_self_local = GAME_STATUS["players"][variables.player_name]
set_data(player_self, player_self_local, fields_self)
def train():
cfg = neural_train
dataset = NeuralData(list_file=cfg['list_file'], data_root=cfg['data_root'])
fw, fh, fd, fc = cfg['fov_shape']
# model = ffn(in_planes=2, module_nums=8)
model = ffn()
print("Initializing weights...")
model.init_weights()
if args.cuda:
model = torch.nn.DataParallel(model) # 多卡存在问题, priors加倍了
cudnn.benchmark = True
if args.cuda:
model = model.cuda()
model.train()
optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum,
weight_decay=args.weight_decay)
criterion = BinaryFocalLoss(gamma=2)
print('Using the specified args:')
print(args)
data_loader = data.DataLoader(dataset, args.batch_size,
num_workers=args.num_workers,
shuffle=True, collate_fn=detection_collate,
pin_memory=True)
writer = SummaryWriter('./logs')
epoch = 0
step_index = 0
batch_iterator = None
epoch_size = len(dataset) // args.batch_size
# Loss counters
vis_loss = []
i = 0
for iteration in range(cfg['max_iter']):
# load train data .tif voxel
if (not batch_iterator) or (iteration % epoch_size == 0):
batch_iterator = iter(data_loader)
epoch += 1
if iteration in cfg['lr_steps']:
step_index += 1
adjust_learning_rate(optimizer, args.gamma, step_index)
images, targets = next(batch_iterator)
vis_loss = []
# if args.cuda:
# images = images.to('cuda')
# targets = targets.to('cuda')
# else:
# images = torch.Tensor(images)
# targets = torch.Tensor(targets)
locations = prepare_data(labels=targets, patch_shape=cfg['subvol_shape'])
# 对patch的训练
indices = np.random.permutation(len(locations))
for location in locations[indices]:
# 抽取patch, 同时生成与中心点相对应的监督标签
subvol_data, subvol_labels, relative_loc = patch_subvol(data=images, labels=targets, subvol_shape=cfg['subvol_shape'],
deltas=np.array(cfg['fov_shape'][:3])//2, location=location)
# 与patch相对应的soft二值mask
subvol_mask = mask_subvol(subvol_data.shape, relative_loc)
n, c, w, h, d = subvol_data.shape
# Create FOV dicts, and center locations
V = {(relative_loc[0], relative_loc[1], relative_loc[2])} # set()
queue = Queue()
queue.put([relative_loc[0], relative_loc[1], relative_loc[2]])
# Compute upper and lower bounds
upper = [w - fw // 2, h - fh // 2, d - fd // 2]
lower = [fw // 2, fh // 2, fd // 2]
p_weights = []
optimizer.zero_grad()
cnt = 0
while not queue.empty():
if cnt > 10:
break
cnt += 1
# Get new list of FOV locations
current_loc = np.array(queue.get(), np.int32)
# Center around FOV
fov_data = get_data(subvol_data, current_loc, cfg['fov_shape'])
fov_labels = get_data(subvol_labels, current_loc, cfg['fov_shape'])
# fov_labels = np.squeeze(fov_labels, axis=1)
fov_mask = get_data(subvol_mask, current_loc, cfg['fov_shape'])
# Loss-weighted
weights = get_weights(fov_labels)
p_weights.append(weights)
# print("weights:", weights)
# criterion = nn.BCEWithLogitsLoss(pos_weight=0.005*weights)
# Add merging of old and new mask
d_m = np.concatenate([fov_data, fov_mask], axis=1)
if args.cuda:
d_m = torch.Tensor(d_m).to('cuda')
fov_labels = torch.Tensor(fov_labels).to('cuda')
else:
d_m = torch.Tensor(d_m)
fov_labels = torch.Tensor(fov_labels)
pred = model(d_m)
# print(type(pred), pred.type())
# print(torch.from_numpy(fov_mask).type())
logit_seed = torch.add(torch.from_numpy(fov_mask).to('cuda'), other=pred)
# logit_seed = pred
prob_seed = expit(logit_seed.detach().cpu().numpy())
if len(vis_loss) % 10 == 0:
# print(np.max(prob_seed), np.min(prob_seed), np.sum(prob_seed>0.95)/(17*17*17))
writer.add_scalars("prob_map", {"max": np.max(prob_seed),
"min": np.min(prob_seed),
"pos_ratio": np.sum(prob_seed>0.95)/(33*33*33),
"1/weights": 1/weights}, i)
# Loss, Backprop
# optimizer.zero_grad()
# print(torch.max(pred), torch.min(pred))
# print(torch.max(torch.sigmoid(pred)), torch.min(torch.sigmoid(pred)))
loss0 = criterion(logit_seed, fov_labels, weights)
loss0.backward(retain_graph=True)
# gradClamp(model.parameters())
# log
if i % 10 == 0:
writer.add_scalars("Train/Loss", {"loss": loss0.data}, i)
for name, layer in model.named_parameters():
writer.add_histogram(name+'_grad', layer.grad.cpu().data.numpy(), i)
writer.add_image("Target", trans3Dto2D(fov_labels.cpu()), i)
writer.add_image("ProbMap", trans3Dto2D(prob_seed), i)
i += 1
vis_loss.append(loss0.detach().item())
if len(vis_loss) % 10 == 0:
print("%d of a tif, FOV Loss: %.6f" % (len(vis_loss), loss0.data.item()))
# 更新patch对应的soft二值mask
set_data(subvol_mask, current_loc, logit_seed.detach().cpu().numpy())
# Compute new locations
new_locations = get_new_locs(logit_seed.detach().cpu().numpy(), cfg['delta'], cfg['tmove'])
for new in new_locations:
new = np.array(new, np.int32) + current_loc
bounds = [lower[j] <= new[j] < upper[j] for j in range(3)]
stored_loc = tuple([new[i] for i in range(3)])
if all(bounds) and stored_loc not in V:
V.add(stored_loc)
queue.put(new)
# mask = subvol_mask >= logit(0.6)
loss1 = len(p_weights) * criterion(torch.Tensor(subvol_mask), torch.Tensor(subvol_labels), np.mean(p_weights))
loss0.data.zero_()
loss0.data = loss1.data
loss0.backward()
optimizer.step()
print("One patch ends of Iteration(%d)/Epoch(%d)" % (iteration, epoch))
print("One tif ends of Iteration(%d)/Epoch(%d)" % (iteration, epoch))
if iteration % 10 == 0:
print('iter ' + repr(iteration) + ' || Loss: %.4f ||' % (loss0.data.item()), end='\n')
# print('timer: %.4f sec. %.4f sec.' % (t2 - t1, t1 - t0))
# if args.visdom:
# update_vis_plot(iteration, min(500, np.mean(vis_loss)), iter_plot, epoch_plot, 'append')
if iteration != 0 and iteration % 20 == 0:
print('Saving state, iter:', iteration)
torch.save(model.state_dict(), args.save_folder +'/FFN_' + dataset.name + "_" +
repr(iteration) + '.pth')
torch.save(model.state_dict(),
args.save_folder + '/FFN_' + dataset.name + '.pth')
def metadata(self, output_dir: str, pyv: str) -> t.Tuple[dict, dict]:
"""Setup templates context"""
_paths: t.Dict[str, str] = {}
_tags: dict = defaultdict()
if os.geteuid() == 0:
log.warning("Detected running as ROOT. Make sure not to "
"use --generate dockerfiles while using "
"manager_images. If you need to regenerate "
"Dockerfiles use manager_dockerfiles instead.")
for pkg in self.packages.keys():
log.info(f"Generate context for {pkg} python{pyv}")
for (release,
distro_version) in self.aggregate_dists_releases(pkg):
log.debug(f"Tag context for {release}:{distro_version}")
# setup our build context
_tag_ctx = self.dockerfiles(pkg, distro_version, pyv)
template_dir = Path(_tag_ctx["templates_dir"])
# generate our image tag.
_release_tag, _build_tag = self.image_tags(
python_version=pyv,
release_type=release,
suffixes=_tag_ctx["add_to_tags"],
)
# setup image tags.
tag_keys = f"{_tag_ctx['package']}:{_release_tag}"
# output path.
base_output_path = Path(output_dir, pkg, f"{distro_version}")
if release != "base":
output_path = Path(base_output_path, release)
else:
output_path = base_output_path
# input path.
input_paths = [
str(f) for f in walk(template_dir) if release in f.name
]
if "rhel" in template_dir.name:
if "cudnn" in release:
input_paths = [
str(f) for f in template_dir.iterdir()
if DOCKERFILE_NVIDIA_REGEX.match(f.name)
]
_tag_ctx.update(
{"docker_build_path": str(output_path)})
full_output_path = str(Path(output_path, DOCKERFILE_NAME))
# # setup directory correspondingly, and add these paths
set_data(_tags, _tag_ctx.copy(), tag_keys)
set_data(_tags, input_paths, tag_keys, "input_paths")
set_data(_tags, str(output_path), tag_keys, "output_path")
if release != "base":
set_data(_tags, _build_tag, tag_keys, "build_tags")
# setup generated path with tag releases
set_data(_paths, full_output_path, tag_keys)
return _tags, _paths
ap.add_argument('--save_dir', dest="save_dir", action="store", default="checkpoint.pth")
args = ap.parse_args()
architecture = args.arch
learning_rate = args.learning_rate
dropout = args.dropout
epochs = args.epochs
hidden_units = args.hidden_units
gpu_enabled = args.gpu_enabled
path = args.save_dir
print(architecture,learning_rate,dropout,epochs,hidden_units,gpu_enabled,path)
train_dataloader, valid_dataloader, test_dataloader, train_datasets= utils.set_data()
model, criterion, optimizer, device = utils.set_up(architecture,hidden_units,learning_rate,dropout,gpu_enabled)
model, optimizer = utils.train_model(epochs,dropout, model, criterion, optimizer, device, train_dataloader, valid_dataloader)
def accuracy():
correct = 0
total = 0
model.eval()
with torch.no_grad():
for images, labels in test_dataloader: