def get_variants_as_products(product_id=None, fieldset=[], product_get_params={}, variant_get_params={}): product_set = [] logger.info( f'Downloading product{"s" if product_id is None else f" {product_id}"}' ) products = get_product(product_id, **product_get_params) for pd in products: prod_id = pd['id'] if 'type' in pd and pd['type'] == 'variable': logger.info(f"Downloading variants of product '{prod_id}'") variants = get_product_variants(prod_id, **variant_get_params) for pvd in variants: product = Product() product.fields = fieldset for field in product.fields: if not isinstance(field, VariantAttributeField): if field.is_variant_field(): field.value_from_dict(pvd) else: field.value_from_dict(pd) else: field.value_from_dict(pvd, pd) product_set.append(product) else: product = Product() product.fields = fieldset for field in product.fields: field.value_from_dict(pd) product_set.append(product) return product_set
async def incorrect_command(message: Message, state: FSMContext): logger.info( f"[{await state.get_state()}] Команда {message.text} определена как ошибочная" ) await message.reply( 'Неверная команда. \nПожалуйста, выберите действие, используя клавиатуру ниже.' )
async def today(message: types.Message, state: FSMContext): """ Show drunk / norm in % """ logger.info(f"today handler {message.text}") abot = await get_jedy(message.from_user.id, state) vol = abot.total() await message.answer(replicas['today'] + str(vol) + '%')
async def add_drunk(message: types.Message, state: FSMContext): """ add one mark of drunk """ logger.info(f"text handler {message.text}") abot = await get_jedy(message.from_user.id, state) abot.add(message.text) sticker = yes_sticker if abot.check() else choice(rand_glass_stickers) await bot.send_sticker(message.from_user.id, sticker)
def test(result_dir): """ upsample a point cloud """ pytorch_utils.load_network(net, CKPT) net.to(DEVICE) net.eval() test_files = glob(TEST_DATA, recursive=True) for point_path in test_files: folder = os.path.basename(os.path.dirname(point_path)) out_path = os.path.join(result_dir, folder, point_path.split('/')[-1][:-4] + '.ply') data = pc_utils.load(point_path, NUM_SHAPE_POINT) data = data[np.newaxis, ...] num_shape_point = data.shape[1] * FLAGS.drop_out if FLAGS.drop_out < 1: _, data = operations.furthest_point_sample( data, int(num_shape_point)) # normalize "unnecessarily" to apply noise data, centroid, furthest_distance = pc_utils.normalize_point_cloud( data) is_2D = np.all(data[:, :, 2] == 0) if JITTER: data = pc_utils.jitter_perturbation_point_cloud( data, sigma=FLAGS.jitter_sigma, clip=FLAGS.jitter_max, is_2D=is_2D) # transpose to NCHW format data = torch.from_numpy(data).transpose(2, 1).to(device=DEVICE) logger.info(os.path.basename(point_path)) start = time.time() with torch.no_grad(): # 1x3xN input_pc_list, pred_pc_list = pc_prediction( net, data, patch_num_ratio=PATCH_NUM_RATIO) # for i, patch_pair in enumerate(zip(input_pc_list, pred_pc_list)): # in_patch, out_patch = patch_pair # pc_utils.save_ply(in_patch.transpose(2, 1).cpu().numpy()[ # 0], path[:-4]+'_input_%d.ply' % i) # pc_utils.save_ply(out_patch.transpose(2, 1).cpu().numpy()[ # 0], path[:-4]+'_output_%d.ply' % i) pred_pc = torch.cat(pred_pc_list, dim=-1) input_point = torch.cat(input_pc_list, dim=-1) end = time.time() print("total time: ", end - start) _, pred_pc = operations.furthest_point_sample( pred_pc, int(num_shape_point) * UP_RATIO, NCHW=True) pred_pc = pred_pc.transpose(2, 1).cpu().numpy() pred_pc = (pred_pc * furthest_distance) + centroid data = data.transpose(2, 1).cpu().numpy() data = (data * furthest_distance) + centroid data = data[0, ...] pred_pc = pred_pc[0, ...] pc_utils.save_ply(data, out_path[:-4] + '_input.ply') pc_utils.save_ply(pred_pc, out_path[:-4] + '.ply')
async def today(message: types.Message, state: FSMContext): """ add one glass (250 ml) """ logger.info(f"glass handler {message.text}") abot = await get_jedy(message.from_user.id, state) abot.add('250') sticker = yes_sticker if abot.check() else choice(rand_glass_stickers) await bot.send_sticker(message.from_user.id, sticker)
def send_mysql(name, ftype, public_time, down_url, connect, logger): '''将数据写入数据库''' sql = "insert into piaohua(name, content, type, updatetime) value ('%s', '%s', '%s', '%s')" % ( name, down_url, ftype, public_time) code = connect.change_data(sql) if code == 0: logger.info('[%s] ok' % name) else: logger.error('[%s] error,message: [%s]' % (name, code))
async def send_welcome(message: types.Message, state: FSMContext): """Show hello message to help with bot""" logger.info(f"Send welcome handler text: {message.text} !") abot = await get_jedy(message.from_user.id, state) logger.info(f"is new user? {abot.new_user(message.from_user.username)}") kb_markup = types.ReplyKeyboardMarkup(row_width=2) btns_text = ('/glass', '/total') kb_markup.row(*(types.KeyboardButton(text) for text in btns_text)) await message.answer(replicas['start'], reply_markup=kb_markup)
async def norm(message: types.Message, state: FSMContext): """ Set norm parameter to X """ logger.info(f"norm handler {message.text}") abot = await get_jedy(message.from_user.id, state) try: x = int(message.text.split()[1]) except ValueError: await message.answer(replicas['wrong_norm']) return except IndexError: return abot.update_norm(x) await message.answer(replicas['norm'])
def load_patch_data(self, h5_path, up_ratio, step_ratio, num_point): """ read point inputs and ground truth from h5 file into memory. h5 file name is train_{dataset1}_{dataset2}.hdf5 dataset names are composed of {label}_{num_point} :param h5_path: string to h5_path up_ratio: integer upscaling ratio step_ratio: integer upscaling ratio of each step num_point: number of points in the input shape :return data: BxNx3 float label: dict label["x{ratio}"] Bx(ratio*N)x3 """ h5_filepath = os.path.join(h5_path) num_points = re.findall("\d+", os.path.basename(h5_filepath)[:-5]) num_points = list(map(int, num_points)) num_points.sort() num_points = np.asarray(num_points) num_in_point = num_points[np.searchsorted(num_points, num_point)] f = h5py.File(h5_filepath, "r") tag = re.findall("_([A-Za-z]+)_", os.path.basename(h5_filepath))[-1] data = f[tag + "_%d" % num_in_point][:, :, 0:3] logger.info("input point_num %d" % data.shape[1]) centroid = np.mean(data[:, :, 0:3], axis=1, keepdims=True) data[:, :, 0:3] = data[:, :, 0:3] - centroid furthest_distance = np.amax(np.sqrt(np.sum(data[:, :, 0:3]**2, axis=-1)), axis=1, keepdims=True) data[:, :, 0:3] = data[:, :, 0:3] / \ np.expand_dims(furthest_distance, axis=-1) label = {} for x in range(1, int(log(up_ratio, step_ratio) + 1)): r = step_ratio**x closest_larger_equal = num_points[np.searchsorted( num_points, num_in_point * r)] label["x%d" % r] = f[tag + "_%d" % closest_larger_equal][:, :, :3] label["x%d" % r][:, :, 0:3] = label["x%d" % r][:, :, 0:3] - centroid label["x%d" % r][:, :, 0:3] = label["x%d" % r][:, :, 0:3] / \ np.expand_dims(furthest_distance, axis=-1) logger.info("gt (ratio %d), point_num %d " % (r, label["x%d" % r].shape[1])) f.close() if np.all(data[:, :, 2] == 0): self.is_2D = True logger.info("2D dataset") logger.info(("total %d samples" % (data.shape[0]))) return data, label
def vis(result_dir): """ upsample a point cloud """ from sklearn.manifold import TSNE from utils.interactive_visualizer import Painter # loaded_states = np.load(CKPT).item() # net.load_state_dict(loaded_states) # pytorch_utils.save_network(net, os.path.dirname(CKPT), "final", "poisson") pytorch_utils.load_network(net, CKPT) net.to(DEVICE) net.eval() test_files = glob(TEST_DATA, recursive=True) for point_path in test_files: folder = os.path.basename(os.path.dirname(point_path)) out_path = os.path.join(result_dir, folder, point_path.split('/')[-1][:-4] + '.ply') data = pc_utils.load(point_path, NUM_SHAPE_POINT) data = data[np.newaxis, ...] num_shape_point = data.shape[1] * FLAGS.drop_out # transpose to NCHW format data = torch.from_numpy(data).transpose(2, 1).to(device=DEVICE) logger.info(os.path.basename(point_path)) start = time.time() with torch.no_grad(): # 1x3xN xyz_dictlist, feat_dictlist, nnIdx_dictlist = pc_visualization( net, data, patch_num_ratio=PATCH_NUM_RATIO) for k, v in nnIdx_dictlist.items(): xyz = xyz_dictlist[k] for p in range(1, len(v)): # v shape is 1xNxK v[p] += v[p - 1].shape[1] xyz = torch.cat(xyz, dim=-1) xyz = xyz.transpose(2, 1).cpu().numpy()[0, ...] nnIdx = torch.cat(v, dim=1) nnIdx = nnIdx.cpu().numpy()[0, ...] painter = Painter("NN Feature") painter.nnIdx = nnIdx painter.interactive_3D_plot(xyz, k)
async def shutdown(dp: Dispatcher): await dp.storage.close() await dp.storage.wait_closed() logger.info('Storage is closed!')
def train(): net.to(DEVICE) net.train() model = Model(net, "train", FLAGS) # data loader if TRAIN_H5 is not None: from data import H5Dataset dataset = H5Dataset( h5_path=TRAIN_H5, num_shape_point=NUM_SHAPE_POINT, num_patch_point=NUM_POINT, batch_size=BATCH_SIZE, up_ratio=UP_RATIO, step_ratio=STEP_RATIO) dataloader = data.DataLoader( dataset, batch_size=1, pin_memory=True, num_workers=4) start_epoch = model.step // len(dataloader) # whenever progress is changed, we need to update: # 1. chamferloss threshold # 2. dataset.combined # 3. dataset.curr_threshold stage, progress = get_stage_progress(model.step) start_ratio = STEP_RATIO ** (stage + 1) dataset.set_max_ratio(start_ratio) if progress > 0.5: dataset.set_combined() if progress > 0.6: model.chamfer_criteria.set_threshold(CD_THRESHOLD) else: model.chamfer_criteria.unset_threshold() dataset.unset_combined() dataloader = data.DataLoader(dataset, batch_size=1, pin_memory=True) # visualization vis_logger = visdom.Visdom(env=FLAGS.id) for epoch in range(start_epoch + 1, MAX_EPOCH): for i, examples in enumerate(dataloader): input_pc, label_pc, ratio = examples ratio = ratio.item() # 1xBx3xN input_pc = input_pc[0].to(DEVICE) label_pc = label_pc[0].to(DEVICE) model.set_input(input_pc, ratio, label_pc=label_pc) # run gradient decent and increment model.step model.optimize() new_stage, new_progress = get_stage_progress(model.step) # advance to the next training stage with an added ratio if stage + 1 == new_stage: dataset.add_next_ratio() dataset.unset_combined() model.chamfer_criteria.unset_threshold() # advance to the combined stage if progress <= 0.5 and new_progress > 0.5: dataset.set_combined() # chamfer loss set ignore threshold if new_progress > 0.6: model.chamfer_criteria.set_threshold(CD_THRESHOLD) if model.step % 50 == 0: output = model.predicted.transpose(2, 1)[0].cpu() gt = model.gt.transpose(2, 1)[0].cpu() input_pc = input_pc.transpose(2, 1)[0].cpu() vis_logger.scatter(input_pc, win="x{}_input".format(ratio), opts=dict(title="x{}_input".format(ratio), markersize=2)) vis_logger.scatter(output, win="x{}_output".format(ratio), opts=dict(title="x{}_output".format(ratio), markersize=2)) vis_logger.scatter(gt, win="x{}_gt".format(ratio), opts=dict(title="x{}_label".format(ratio), markersize=2)) vis_logger.line( np.array([model.error_log["cd_loss_x{}".format(ratio)]]), np.array([model.step]), update="append", win="x{}_loss".format(ratio), opts=dict(title="x{}_loss".format(ratio))) stage, progress = new_stage, new_progress # end of epoch logger.info("epoch %d: " % epoch + ", ".join(["{}={}".format(k, v) for k, v in model.error_log.items()])) if epoch % 20 == 0: pytorch_utils.save_network(net, MODEL_DIR, "model", epoch_label=str(epoch), step=str(model.step))
def writeprogress(self): filename=config.PROGRESS_LOG#+str(datetime.now()) with open(filename,'w') as f: f.write("%d,%d" % (config.TOTAL_COMPLETED+self.donecount,config.TOTAL_COMPLETED+(self.threads_running()+self.notstartedcount+self.donecount))) logger.info("%d,%d" % (config.TOTAL_COMPLETED+self.donecount,config.TOTAL_COMPLETED+(self.threads_running()+self.notstartedcount+self.donecount))+" " + str(datetime.now()))
cass.apply_settings(generate_cassio_conf()) cass.set_default_region(config["region"]) if "api-key" in config: cass.set_riot_api_key(config["api-key"]) manager = Manager() sqlstore = get_sqlstore(cass) from cassiopeia.core import League, ChallengerLeague, MasterLeague, Summoner from cassiopeia.data import Queue if config["seed"]["type"] == "LEAGUE": # Load this league and use it as seeding league = None if config["seed"]["id"] == "CHALLENGER": logger.info("Using challenger league as seed") league = ChallengerLeague(queue=Queue.ranked_solo_fives) elif config["seed"]["id"] == "MASTER": logger.info("Using master league as seed") league = MasterLeague(queue=Queue.ranked_solo_fives) else: logger.info("Using league with id=" + config["seed"]["id"] + " as seed") league = League(id=config["seed"]["id"]) for entry in league.entries: summoner = entry.summoner manager.add_summoner(summoner) logger.info("Loaded " + str(manager.summoner_len) + " summoners") for i in range(config["threads"]): crawler = Crawler(manager, sqlstore)