def update(self, try_reconnect=False): providers = utils.get_available_providers() channels = [] for sl in providers: logger.log.info('Getting channels for account: %s' % (sl.account.username)) try: channels = channels + sl.channels() except skylink.TooManyDevicesException as e: if self._addon.getSetting('reuse_last_device') == 'true': device = utils.get_last_used_device(e.devices) else: device = utils.select_device( e.devices) if try_reconnect else '' if device != '': logger.log.info('reconnecting as: ' + device) sl.reconnect(device) channels = channels + sl.channels() else: raise # Remove duplicate channels (same channels provided by skylink.sk and skylink.cz) if bool(self._addon.getSetting('playlist_unique')): logger.log.info('Filtering channels to create unique list...') channels = utils.unique_channels(channels) logger.log.info('Updating playlist [%d channels]' % len(channels)) try: path = os.path.join(self._addon.getSetting('playlist_folder'), self._addon.getSetting('playlist_file')) _skylink_logos = 'true' == self._addon.getSetting('sl_logos') exports.create_m3u(channels, path, sl.getUrl() + '/' if _skylink_logos else None) result = 1 except IOError as e: logger.log.error(str(e)) raise skylink.SkylinkException(30503) if bool(self._addon.getSetting('epg_generate')): days = int(self._addon.getSetting('epg_days')) logger.log.info('Updating EPG [%d days from %s]' % (days, datetime.datetime.now())) path = os.path.join(self._addon.getSetting('epp_folder'), self._addon.getSetting('epg_file')) try: exports.create_epg( channels, sl.epg(channels, datetime.datetime.now(), days), path, self._addon, sl.getUrl() + '/') result = 2 except IOError as e: logger.log.error(str(e)) raise skylink.SkylinkException(30504) return result
def _parse(self, kwargs): state_dict = self._state_dict() for k, v in kwargs.items(): if k not in state_dict: raise ValueError('UnKnown Option: "--%s"' % k) setattr(self, k, v) self.device, self.gpu_id = select_device() print('======user config========') pprint(self._state_dict()) print('==========end============')
def visualize(data_size=10000): ''' Plot 3-dimensional feature space and reconstructed result. Images are stored in 'result_SDAE' * Have to used autoencoder models with 3-dimensional output Can only be used on training data * ''' batchSize = 100 device = utils.select_device() Test_dataset = utils.StateData(data_size=data_size) visu_loader = torch.utils.data.DataLoader(Test_dataset, batch_size=batchSize, shuffle=True) chekp = torch.load('model/chekp.pt') reconstruct_dim = chekp['in_dim'] feature_dim = chekp['out_dim'] chekp_model = chekp['model'] stack_num = chekp['stack_num'] model = StackDAE(reconstruct_dim, feature_dim, stack_num) model.to(device).load_state_dict(chekp_model) reconstruct_stack = torch.FloatTensor(reconstruct_dim).to( device).unsqueeze(0) feature_stack = torch.FloatTensor(feature_dim).to(device).unsqueeze(0) for i, data in enumerate(visu_loader): data = data.to(device) reconstruct = model.forward(data) reconstruct_stack = torch.cat((reconstruct_stack, reconstruct)) feature_stack = torch.cat((feature_stack, model.hidden_feature)) reconstruct_stack = reconstruct_stack[1:].detach().cpu().numpy() feature_stack = feature_stack[1:].detach().cpu().numpy() print('%d data points, input size: %d, feature size: %d' % (feature_stack.shape[0], reconstruct_dim, feature_stack.shape[1])) fig1 = plt.figure() ax = Axes3D(fig1) ax.scatter(feature_stack[:, 0], feature_stack[:, 1], feature_stack[:, 2]) fig1.savefig('result_SDAE/result_feature.png') fig2 = plt.figure() ax = Axes3D(fig2) ax.scatter(reconstruct_stack[:, -8], reconstruct_stack[:, -7], reconstruct_stack[:, -6]) fig2.savefig('result_SDAE/result_reconstruct.png') return feature_stack
def main(args=None): args = parse_args(args) select_device(0 if torch.cuda.is_available() and not args.cpu else -1) #select_device(-1) logger.prog('device: {}'.format(Config.DEVICE)) if args.task == 'learn': exp_f = learning elif args.task == 'cost': exp_f = compare_cost elif args.task == 'grad': exp_f = compare_grad else: raise Exception('unsupported task') if args.post_f is not None: post_f = lambda results: getattr(postprocess, args.post_f)(args, results) else: post_f = None if args.mode == 'single': exp_f(args) elif args.mode == 'seeds': running_seeds(args.save_fn, exp_f, argparse.Namespace(**vars(args)), args.n_seeds, post_f=post_f) elif args.mode == 'collect': assert args.task in ['grad', 'learn'] success_f = lambda result: len(result[-1]['out']) == 0 collect_seeds(args.save_fn, exp_f, args, success_f=success_f, n_seeds=args.n_seeds, max_seed=args.max_seed, post_f=post_f)
def _set_cuda(self, model): has_device = hasattr(self.config, "device") if has_device: device = self.config.device else: device = "" device = select_device(device, self.config.train.batch_size) model = torch.nn.DataParallel(model).to(device) self.device = device self.cuda = self.device.type != 'cpu' setattr(self.config, "device", device) setattr(self.config, "cuda", self.cuda) return model
def visualize_orig(data_size=10000): ''' Plot positions(x, y, z) of the last joint of original data(observation). * Can only be used on training data * ''' batchSize = 100 device = utils.select_device() Test_dataset = utils.StateData(data_size) visu_loader = torch.utils.data.DataLoader(Test_dataset, batch_size=batchSize, shuffle=True) output_stack = torch.FloatTensor(48).unsqueeze(0) for i, data in enumerate(visu_loader): output_stack = torch.cat((output_stack, data)) output_stack = output_stack[1:].numpy() fig = plt.figure() ax = Axes3D(fig) ax.scatter(output_stack[:, -8], output_stack[:, -7], output_stack[:, -6]) fig.savefig('result_SDAE/result_orig.png')
def evaluate(cfg, args): device = select_device(args.device) # Initialize/load model if cfg.MODEL.META_ARCHITECTURE: # Initialize model model = YOLOv3(cfg).to(device) # Load weights if cfg.TEST.WEIGHTS.endswith(".pth"): state = torch.load(cfg.TEST.WEIGHTS, map_location=device) model.load_state_dict(state["state_dict"]) else: load_darknet_weights(model, cfg.TEST.WEIGHTS) if device.type != "cpu" and torch.cuda.device_count() > 1: model = nn.DataParallel(model) else: warnings.warn( "WARNNING: Backbone network cannot be empty! " f"Default load Darknet53 meta architecture for `{cfg.CONFIG_FILE}`!" ) model = YOLOv3(cfg).to(device) if cfg.TEST.TASK == "visual": images = os.listdir(os.path.join(os.getcwd(), "data", "test")) for filename in images: path = os.path.join( os.path.join(os.getcwd(), "data", "test", filename)) images = cv2.imread(path) assert images is not None bboxes_prd = Evaluator(model, cfg=cfg).get_bbox(images) if bboxes_prd.shape[0] != 0: boxes = bboxes_prd[..., :4] class_inds = bboxes_prd[..., 5].astype(np.int32) scores = bboxes_prd[..., 4] visualize_boxes(image=images, boxes=boxes, labels=class_inds, probs=scores, class_labels=cfg.CLASSES) path = os.path.join(f"./outputs/{filename}") cv2.imwrite(path, images) elif cfg.TEST.TASK == "eval": maps = 0. with torch.no_grad(): aps = Evaluator(model, visiual=True, cfg=cfg).calculate_aps(cfg.TEST.MULTI_SCALE, cfg.TEST.FLIP) for i in aps: print(f"{i:25s} --> mAP : {aps[i]:.4f}") maps += aps[i] maps = maps / len(cfg.CLASSES) print(f'mAP:{maps:.6f}') return maps
def train(): input_dir, var1, var2, adam, device = opt.input, opt.var1, opt.var2, opt.adam, opt.device data = Dataset(input_dir).data device = utils.select_device(device, batch_size=opt.batch_size) for i in (var1, var2): variable_check(data, i) use_cuda = torch.cuda.is_available() X_reshape = data[var1].values.reshape(-1, 1) y_reshape = data[var2].values.reshape(-1, 1) if use_cuda: x, y = gpu_dataset(X_reshape, y_reshape) else: x, y = cpu_dataset(X_reshape, y_reshape) # Initialize model net = LinearNet(n_feature=x.size(1), n_output=y.size(1)).to(device) if adam: # optimizer using Adam optimizer = torch.optim.Adam(net.parameters(), lr=0.001) else: # optimizer using SGD optimizer = torch.optim.SGD(net.parameters(), lr=0.001) loss_func = nn.MSELoss() batch_size = opt.batch_size n_epochs = opt.epoch batch_no = len(x) // batch_size train_loss = 0 train_loss_min = np.Inf if use_cuda: for epoch in range(n_epochs): for i in range(batch_no): start = i * batch_size end = start + batch_size optimizer.zero_grad() prediction = net(x) loss = loss_func(prediction, y) loss.backward() optimizer.step() values, labels = torch.max(prediction, 1) num_right = np.sum(labels.cpu().data.numpy() == y[start:end]) train_loss += loss.item() * batch_size train_loss = train_loss / len(x) if train_loss <= train_loss_min: print( "Validation loss decreased ({:6f} ===> {:6f}). Saving the model..." .format(train_loss_min, train_loss)) torch.save(net.state_dict(), "regression_model.pt") train_loss_min = train_loss if epoch % 50 == 0: print('') print("Epoch: {} \tTrain Loss: {} \tTrain Accuracy: {}".format( epoch + 1, train_loss, num_right / len(y[start:end]))) print('Training Ended! ')
def update(self, try_reconnect=False): result = -1 _playlist_generate = 'true' == self._addon.getSetting( 'playlist_generate') _epg_generate = 'true' == self._addon.getSetting('epg_generate') if not _playlist_generate and not _epg_generate: return result _username = self._addon.getSetting('username') _password = self._addon.getSetting('password') _profile = xbmc.translatePath(self._addon.getAddonInfo('profile')) _provider = 'directone.hu' _pin_protected_content = 'false' != self._addon.getSetting( 'pin_protected_content') sl = skylink.Skylink(_username, _password, _profile, _provider, _pin_protected_content) logger.log.info('SL created') try: channels = sl.channels() except skylink.TooManyDevicesException as e: if self._addon.getSetting('reuse_last_device') == 'true': device = utils.get_last_used_device(e.devices) else: device = utils.select_device( e.devices) if try_reconnect else '' if device != '': logger.log.info('reconnecting as: ' + device) sl.reconnect(device) channels = sl.channels() else: raise if _playlist_generate: try: path = os.path.join(self._addon.getSetting('playlist_folder'), self._addon.getSetting('playlist_file')) _skylink_logos = 'true' == self._addon.getSetting('sl_logos') logger.log.info('Updating playlist [%d channels]' % len(channels)) exports.create_m3u(channels, path, sl._api_url if _skylink_logos else None) result = 1 except IOError as e: logger.log.error(str(e)) raise skylink.SkylinkException(30503) if _epg_generate: try: days = int(self._addon.getSetting('epg_days')) catchup_days = int(self._addon.getSetting('epg_days_catchup')) today = datetime.datetime.now() epgFrom = today - datetime.timedelta( days=catchup_days) if catchup_days > 0 else today epgTo = today + datetime.timedelta(days=days) path = os.path.join(self._addon.getSetting('epp_folder'), self._addon.getSetting('epg_file')) logger.log.info('Updating EPG [from %s to %s]' % (epgFrom, epgTo)) exports.create_epg(channels, sl.epg(channels, epgFrom, epgTo), path) result = 2 except IOError as e: logger.log.error(str(e)) raise skylink.SkylinkException(30504) return result
print(f"{epoch - start_epoch} epochs completed " f"in {(time.time() - start_time) / 3600:.3f} hours.\n") dist.destroy_process_group() if torch.cuda.device_count() > 1 else None torch.cuda.empty_cache() if __name__ == "__main__": mp.set_start_method("spawn", force=True) args = get_parser().parse_args() cfg = setup_cfg(args) args.weights = "weights/checkpoint.pth" if args.resume else args.weights print(args) device = select_device(args.device, apex=mixed_precision) if device.type == "cpu": mixed_precision = False try: os.makedirs("weights") except OSError: pass try: # Start Tensorboard with "tensorboard --logdir=runs" from torch.utils.tensorboard import SummaryWriter tb_writer = SummaryWriter() except:
default='/u01/data/bdd100k/det/bdd100k.yaml', help='data.yaml path') parser.add_argument('--epochs', type=int, default=1) parser.add_argument("--device", type=str, default='cpu') parser.add_argument("--batch_size", type=int, default=32) parser.add_argument("--input_size", type=int, default=640) parser.add_argument('--conf_thres', type=float, default=0.001) parser.add_argument('--iou_thres', type=float, default=0.6) parser.add_argument('--augment', action='store_true') parser.add_argument('--verbose', action='store_true') parser.add_argument('--save_txt', action='store_true') parser.add_argument('--save_conf', action='store_true') parser.add_argument('--save_json', action='store_true') parser.add_argument('--single_cls', action='store_true') opt = parser.parse_args() device = select_device(opt.device) """ Init model """ od = DetectionModel(opt.model_name, opt.model_version, opt) """ Init data """ with open(opt.data) as f: data = yaml.load(f, Loader=yaml.FullLoader) # model dict path = data['test'] nc = 1 if opt.single_cls else int(data['nc']) # number of classes iouv = torch.linspace(0.5, 0.95, 10).to(device) # iou vector for [email protected]:0.95 niou = iouv.numel() dataloader = create_dataloader(path,
# return CosineAnnealingLR(optim, t_max) elif sche_type == "auto": return LambdaLR(optim, lambda x: (((1 + np.cos(x * np.pi / epochs)) / 2) ** 1.0) * 0.9 + 0.1) else: return None if __name__ == "__main__": params_file = 'params.yml' params_file = check_file(params_file) params = Params('params.yml') # load params params.save_dir = os.path.join(os.getcwd(), params.save_dir) os.makedirs(params.save_dir, exist_ok=True) # build ouput directory device = select_device(params.device, batch_size=params.batch_size) # build GPU env init_seeds(1) train_loader, val_loader = get_loaders(params.input_dir, params.num_classes, params.img_size, params.batch_size, params.num_workers) net, ckpt = get_model(params) net = nn.DataParallel(net).to(device, non_blocking=True) ''' This CrossEntropyLoss implementation comes with a softmax activation function, which is not suitable for this multi-label classification situation ''' # loss = nn.CrossEntropyLoss() loss = nn.BCEWithLogitsLoss() if params.multilabels else nn.CrossEntropyLoss() ''' Adam optimizer has fastest training speed, but with Familiarity to data, SGD is recommanded