def get_variants_as_products(product_id=None,
fieldset=[],
product_get_params={},
variant_get_params={}):
product_set = []
logger.info(
f'Downloading product{"s" if product_id is None else f" {product_id}"}'
)
products = get_product(product_id, **product_get_params)
for pd in products:
prod_id = pd['id']
if 'type' in pd and pd['type'] == 'variable':
logger.info(f"Downloading variants of product '{prod_id}'")
variants = get_product_variants(prod_id, **variant_get_params)
for pvd in variants:
product = Product()
product.fields = fieldset
for field in product.fields:
if not isinstance(field, VariantAttributeField):
if field.is_variant_field():
field.value_from_dict(pvd)
else:
field.value_from_dict(pd)
else:
field.value_from_dict(pvd, pd)
product_set.append(product)
else:
product = Product()
product.fields = fieldset
for field in product.fields:
field.value_from_dict(pd)
product_set.append(product)
return product_set
async def incorrect_command(message: Message, state: FSMContext):
logger.info(
f"[{await state.get_state()}] Команда {message.text} определена как ошибочная"
)
await message.reply(
'Неверная команда. \nПожалуйста, выберите действие, используя клавиатуру ниже.'
)
async def today(message: types.Message, state: FSMContext):
""" Show drunk / norm in % """
logger.info(f"today handler {message.text}")
abot = await get_jedy(message.from_user.id, state)
vol = abot.total()
await message.answer(replicas['today'] + str(vol) + '%')
async def add_drunk(message: types.Message, state: FSMContext):
""" add one mark of drunk """
logger.info(f"text handler {message.text}")
abot = await get_jedy(message.from_user.id, state)
abot.add(message.text)
sticker = yes_sticker if abot.check() else choice(rand_glass_stickers)
await bot.send_sticker(message.from_user.id, sticker)
def test(result_dir):
"""
upsample a point cloud
"""
pytorch_utils.load_network(net, CKPT)
net.to(DEVICE)
net.eval()
test_files = glob(TEST_DATA, recursive=True)
for point_path in test_files:
folder = os.path.basename(os.path.dirname(point_path))
out_path = os.path.join(result_dir, folder,
point_path.split('/')[-1][:-4] + '.ply')
data = pc_utils.load(point_path, NUM_SHAPE_POINT)
data = data[np.newaxis, ...]
num_shape_point = data.shape[1] * FLAGS.drop_out
if FLAGS.drop_out < 1:
_, data = operations.furthest_point_sample(
data, int(num_shape_point))
# normalize "unnecessarily" to apply noise
data, centroid, furthest_distance = pc_utils.normalize_point_cloud(
data)
is_2D = np.all(data[:, :, 2] == 0)
if JITTER:
data = pc_utils.jitter_perturbation_point_cloud(
data, sigma=FLAGS.jitter_sigma, clip=FLAGS.jitter_max, is_2D=is_2D)
# transpose to NCHW format
data = torch.from_numpy(data).transpose(2, 1).to(device=DEVICE)
logger.info(os.path.basename(point_path))
start = time.time()
with torch.no_grad():
# 1x3xN
input_pc_list, pred_pc_list = pc_prediction(
net, data, patch_num_ratio=PATCH_NUM_RATIO)
# for i, patch_pair in enumerate(zip(input_pc_list, pred_pc_list)):
# in_patch, out_patch = patch_pair
# pc_utils.save_ply(in_patch.transpose(2, 1).cpu().numpy()[
# 0], path[:-4]+'_input_%d.ply' % i)
# pc_utils.save_ply(out_patch.transpose(2, 1).cpu().numpy()[
# 0], path[:-4]+'_output_%d.ply' % i)
pred_pc = torch.cat(pred_pc_list, dim=-1)
input_point = torch.cat(input_pc_list, dim=-1)
end = time.time()
print("total time: ", end - start)
_, pred_pc = operations.furthest_point_sample(
pred_pc, int(num_shape_point) * UP_RATIO, NCHW=True)
pred_pc = pred_pc.transpose(2, 1).cpu().numpy()
pred_pc = (pred_pc * furthest_distance) + centroid
data = data.transpose(2, 1).cpu().numpy()
data = (data * furthest_distance) + centroid
data = data[0, ...]
pred_pc = pred_pc[0, ...]
pc_utils.save_ply(data, out_path[:-4] + '_input.ply')
pc_utils.save_ply(pred_pc, out_path[:-4] + '.ply')
async def today(message: types.Message, state: FSMContext):
""" add one glass (250 ml) """
logger.info(f"glass handler {message.text}")
abot = await get_jedy(message.from_user.id, state)
abot.add('250')
sticker = yes_sticker if abot.check() else choice(rand_glass_stickers)
await bot.send_sticker(message.from_user.id, sticker)
def send_mysql(name, ftype, public_time, down_url, connect, logger):
'''将数据写入数据库'''
sql = "insert into piaohua(name, content, type, updatetime) value ('%s', '%s', '%s', '%s')" % (
name, down_url, ftype, public_time)
code = connect.change_data(sql)
if code == 0:
logger.info('[%s] ok' % name)
else:
logger.error('[%s] error,message: [%s]' % (name, code))
async def send_welcome(message: types.Message, state: FSMContext):
"""Show hello message to help with bot"""
logger.info(f"Send welcome handler text: {message.text} !")
abot = await get_jedy(message.from_user.id, state)
logger.info(f"is new user? {abot.new_user(message.from_user.username)}")
kb_markup = types.ReplyKeyboardMarkup(row_width=2)
btns_text = ('/glass', '/total')
kb_markup.row(*(types.KeyboardButton(text) for text in btns_text))
await message.answer(replicas['start'], reply_markup=kb_markup)
async def norm(message: types.Message, state: FSMContext):
""" Set norm parameter to X """
logger.info(f"norm handler {message.text}")
abot = await get_jedy(message.from_user.id, state)
try:
x = int(message.text.split()[1])
except ValueError:
await message.answer(replicas['wrong_norm'])
return
except IndexError:
return
abot.update_norm(x)
await message.answer(replicas['norm'])
def load_patch_data(self, h5_path, up_ratio, step_ratio, num_point):
"""
read point inputs and ground truth from h5 file into memory.
h5 file name is train_{dataset1}_{dataset2}.hdf5
dataset names are composed of {label}_{num_point}
:param
h5_path: string to h5_path
up_ratio: integer upscaling ratio
step_ratio: integer upscaling ratio of each step
num_point: number of points in the input shape
:return
data: BxNx3 float
label: dict label["x{ratio}"] Bx(ratio*N)x3
"""
h5_filepath = os.path.join(h5_path)
num_points = re.findall("\d+", os.path.basename(h5_filepath)[:-5])
num_points = list(map(int, num_points))
num_points.sort()
num_points = np.asarray(num_points)
num_in_point = num_points[np.searchsorted(num_points, num_point)]
f = h5py.File(h5_filepath, "r")
tag = re.findall("_([A-Za-z]+)_", os.path.basename(h5_filepath))[-1]
data = f[tag + "_%d" % num_in_point][:, :, 0:3]
logger.info("input point_num %d" % data.shape[1])
centroid = np.mean(data[:, :, 0:3], axis=1, keepdims=True)
data[:, :, 0:3] = data[:, :, 0:3] - centroid
furthest_distance = np.amax(np.sqrt(np.sum(data[:, :, 0:3]**2,
axis=-1)),
axis=1,
keepdims=True)
data[:, :, 0:3] = data[:, :, 0:3] / \
np.expand_dims(furthest_distance, axis=-1)
label = {}
for x in range(1, int(log(up_ratio, step_ratio) + 1)):
r = step_ratio**x
closest_larger_equal = num_points[np.searchsorted(
num_points, num_in_point * r)]
label["x%d" % r] = f[tag + "_%d" % closest_larger_equal][:, :, :3]
label["x%d" % r][:, :,
0:3] = label["x%d" % r][:, :, 0:3] - centroid
label["x%d" % r][:, :, 0:3] = label["x%d" % r][:, :, 0:3] / \
np.expand_dims(furthest_distance, axis=-1)
logger.info("gt (ratio %d), point_num %d " %
(r, label["x%d" % r].shape[1]))
f.close()
if np.all(data[:, :, 2] == 0):
self.is_2D = True
logger.info("2D dataset")
logger.info(("total %d samples" % (data.shape[0])))
return data, label
def vis(result_dir):
"""
upsample a point cloud
"""
from sklearn.manifold import TSNE
from utils.interactive_visualizer import Painter
# loaded_states = np.load(CKPT).item()
# net.load_state_dict(loaded_states)
# pytorch_utils.save_network(net, os.path.dirname(CKPT), "final", "poisson")
pytorch_utils.load_network(net, CKPT)
net.to(DEVICE)
net.eval()
test_files = glob(TEST_DATA, recursive=True)
for point_path in test_files:
folder = os.path.basename(os.path.dirname(point_path))
out_path = os.path.join(result_dir, folder,
point_path.split('/')[-1][:-4] + '.ply')
data = pc_utils.load(point_path, NUM_SHAPE_POINT)
data = data[np.newaxis, ...]
num_shape_point = data.shape[1] * FLAGS.drop_out
# transpose to NCHW format
data = torch.from_numpy(data).transpose(2, 1).to(device=DEVICE)
logger.info(os.path.basename(point_path))
start = time.time()
with torch.no_grad():
# 1x3xN
xyz_dictlist, feat_dictlist, nnIdx_dictlist = pc_visualization(
net, data, patch_num_ratio=PATCH_NUM_RATIO)
for k, v in nnIdx_dictlist.items():
xyz = xyz_dictlist[k]
for p in range(1, len(v)):
# v shape is 1xNxK
v[p] += v[p - 1].shape[1]
xyz = torch.cat(xyz, dim=-1)
xyz = xyz.transpose(2, 1).cpu().numpy()[0, ...]
nnIdx = torch.cat(v, dim=1)
nnIdx = nnIdx.cpu().numpy()[0, ...]
painter = Painter("NN Feature")
painter.nnIdx = nnIdx
painter.interactive_3D_plot(xyz, k)
async def shutdown(dp: Dispatcher):
await dp.storage.close()
await dp.storage.wait_closed()
logger.info('Storage is closed!')
def train():
net.to(DEVICE)
net.train()
model = Model(net, "train", FLAGS)
# data loader
if TRAIN_H5 is not None:
from data import H5Dataset
dataset = H5Dataset(
h5_path=TRAIN_H5,
num_shape_point=NUM_SHAPE_POINT, num_patch_point=NUM_POINT,
batch_size=BATCH_SIZE, up_ratio=UP_RATIO, step_ratio=STEP_RATIO)
dataloader = data.DataLoader(
dataset, batch_size=1, pin_memory=True, num_workers=4)
start_epoch = model.step // len(dataloader)
# whenever progress is changed, we need to update:
# 1. chamferloss threshold
# 2. dataset.combined
# 3. dataset.curr_threshold
stage, progress = get_stage_progress(model.step)
start_ratio = STEP_RATIO ** (stage + 1)
dataset.set_max_ratio(start_ratio)
if progress > 0.5:
dataset.set_combined()
if progress > 0.6:
model.chamfer_criteria.set_threshold(CD_THRESHOLD)
else:
model.chamfer_criteria.unset_threshold()
dataset.unset_combined()
dataloader = data.DataLoader(dataset, batch_size=1, pin_memory=True)
# visualization
vis_logger = visdom.Visdom(env=FLAGS.id)
for epoch in range(start_epoch + 1, MAX_EPOCH):
for i, examples in enumerate(dataloader):
input_pc, label_pc, ratio = examples
ratio = ratio.item()
# 1xBx3xN
input_pc = input_pc[0].to(DEVICE)
label_pc = label_pc[0].to(DEVICE)
model.set_input(input_pc, ratio, label_pc=label_pc)
# run gradient decent and increment model.step
model.optimize()
new_stage, new_progress = get_stage_progress(model.step)
# advance to the next training stage with an added ratio
if stage + 1 == new_stage:
dataset.add_next_ratio()
dataset.unset_combined()
model.chamfer_criteria.unset_threshold()
# advance to the combined stage
if progress <= 0.5 and new_progress > 0.5:
dataset.set_combined()
# chamfer loss set ignore threshold
if new_progress > 0.6:
model.chamfer_criteria.set_threshold(CD_THRESHOLD)
if model.step % 50 == 0:
output = model.predicted.transpose(2, 1)[0].cpu()
gt = model.gt.transpose(2, 1)[0].cpu()
input_pc = input_pc.transpose(2, 1)[0].cpu()
vis_logger.scatter(input_pc, win="x{}_input".format(ratio),
opts=dict(title="x{}_input".format(ratio),
markersize=2))
vis_logger.scatter(output, win="x{}_output".format(ratio),
opts=dict(title="x{}_output".format(ratio),
markersize=2))
vis_logger.scatter(gt, win="x{}_gt".format(ratio),
opts=dict(title="x{}_label".format(ratio),
markersize=2))
vis_logger.line(
np.array([model.error_log["cd_loss_x{}".format(ratio)]]),
np.array([model.step]),
update="append",
win="x{}_loss".format(ratio),
opts=dict(title="x{}_loss".format(ratio)))
stage, progress = new_stage, new_progress
# end of epoch
logger.info("epoch %d: " % epoch +
", ".join(["{}={}".format(k, v) for k, v in model.error_log.items()]))
if epoch % 20 == 0:
pytorch_utils.save_network(net, MODEL_DIR,
"model", epoch_label=str(epoch),
step=str(model.step))
def writeprogress(self):
filename=config.PROGRESS_LOG#+str(datetime.now())
with open(filename,'w') as f:
f.write("%d,%d" % (config.TOTAL_COMPLETED+self.donecount,config.TOTAL_COMPLETED+(self.threads_running()+self.notstartedcount+self.donecount)))
logger.info("%d,%d" % (config.TOTAL_COMPLETED+self.donecount,config.TOTAL_COMPLETED+(self.threads_running()+self.notstartedcount+self.donecount))+" " + str(datetime.now()))
cass.apply_settings(generate_cassio_conf())
cass.set_default_region(config["region"])
if "api-key" in config:
cass.set_riot_api_key(config["api-key"])
manager = Manager()
sqlstore = get_sqlstore(cass)
from cassiopeia.core import League, ChallengerLeague, MasterLeague, Summoner
from cassiopeia.data import Queue
if config["seed"]["type"] == "LEAGUE":
# Load this league and use it as seeding
league = None
if config["seed"]["id"] == "CHALLENGER":
logger.info("Using challenger league as seed")
league = ChallengerLeague(queue=Queue.ranked_solo_fives)
elif config["seed"]["id"] == "MASTER":
logger.info("Using master league as seed")
league = MasterLeague(queue=Queue.ranked_solo_fives)
else:
logger.info("Using league with id=" + config["seed"]["id"] +
" as seed")
league = League(id=config["seed"]["id"])
for entry in league.entries:
summoner = entry.summoner
manager.add_summoner(summoner)
logger.info("Loaded " + str(manager.summoner_len) + " summoners")
for i in range(config["threads"]):
crawler = Crawler(manager, sqlstore)
