def call_fibaro_api(url): try: response = requests.get(url, auth=(config.fibaro_user, config.fibaro_password)) except requests.exceptions.ConnectionError: hlp.write_log("ERROR: Connection error for " + url) raise SystemExit("ERROR: Connection error for " + url) return response
def train(): print('start training ...........') batch_size = 32 num_epochs = 600 lr = 0.001 device = torch.device("cuda:0" if (torch.cuda.is_available()) else "cpu") model = SSD_VGG(num_classes=21, device=device, freeze=False) # model.load_state_dict(torch.load('output/weight.pth', map_location=device)) train_loader, val_loader = get_loader(batch_size=batch_size) optimizer = optim.SGD(model.parameters(), lr=lr, momentum=0.9, nesterov=True, weight_decay=0.0005) scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.1, patience=50) criterion = MultiBoxLoss(priors_cxcy=model.priors_cxcy, device=device) train_losses, val_losses = [], [] for epoch in range(num_epochs): train_epoch_loss = fit(epoch, model, optimizer, criterion, device, train_loader, phase='training') val_epoch_loss = fit(epoch, model, optimizer, criterion, device, val_loader, phase='validation') print('-----------------------------------------') if epoch == 0 or val_epoch_loss <= np.min(val_losses): torch.save(model.state_dict(), 'output/weight.pth') # if epoch == 0 or train_epoch_loss <= np.min(train_losses): # torch.save(model.state_dict(), 'output/weight.pth') train_losses.append(train_epoch_loss) val_losses.append(val_epoch_loss) write_figure('output', train_losses, val_losses) write_log('output', epoch, train_epoch_loss, val_epoch_loss) scheduler.step(val_epoch_loss)
def train(root, device, model, epochs, bs, lr): print('start training ...........') train_loader, val_loader = get_loader(root=root, batch_size=bs, shuffle=True) optimizer = optim.Adam(model.parameters(), lr=lr) criterion = HDRLoss(device) train_losses, val_losses = [], [] for epoch in range(epochs): train_epoch_loss = fit(epoch, model, optimizer, criterion, device, train_loader, phase='training') val_epoch_loss = fit(epoch, model, optimizer, criterion, device, val_loader, phase='validation') print('-----------------------------------------') if epoch == 0 or val_epoch_loss <= np.min(val_losses): torch.save(model.state_dict(), 'output/weight.pth') train_losses.append(train_epoch_loss) val_losses.append(val_epoch_loss) write_figures('output', train_losses, val_losses) write_log('output', epoch, train_epoch_loss, val_epoch_loss)
def __init__(self): version = "0.1" # Проверить наличие setting.ini if not os.path.isfile(r"{0}\\settings.ini".format(getcwd())): helper.write_log("Нет файла settings.ini", "error") # Запуск процедуры создания файла основных настроек self.create_global_setting()
def detect_paranada(hist, filename): mean = np.mean(hist) std = np.std(hist) stat_z = [(s-mean)/std for s in hist] paranada = (np.abs(stat_z) > 2) indices = [i for i, x in enumerate(paranada) if x == True] if indices == []: max_hist = max(hist) paranada = (np.abs(np.abs(hist - max_hist) <= 2)) indices = [i for i, x in enumerate(paranada) if x == True] log_message = "WARNING: Failed to get outlier of " + filename helper.write_log('dataset', '4', log_message) # helper.show_plot(i, hist, "") group_paranada = list(helper.split_tol(indices,2)) paranada_index = [i[0] for i in group_paranada] if len(paranada_index) < 5: log_message = "FATAL ERROR: Paranada index of " + filename + " is not completely detected" helper.write_log('dataset', '1', log_message) print("Something error, please check dataset.log!") return paranada, group_paranada, paranada_index
def get_data(obj): hlp.write_log("### Getting data from " + obj + " API: ", eol=False) target_url = config.fibaro_url + obj r = call_fibaro_api(target_url) if r.status_code == 200: save_data(r.json(), obj) else: hlp.write_log("ERROR: error from " + obj + "API code: " + str(r.status_code))
def create_settings(self): path = r'{0}\\settings.ini'.format(getcwd()) try: file = open(path, 'w') file.write('[global]\n') file.write('setup = true\n') file.close() except: helper.write_log('Невозможно создать файл settings.txt')
def export_codebooks(self, filename): f = open(filename + ".csv", "w") for codebook in self.codebooks: for num in codebook[:-1]: f.write(str(num) + ", ") f.write(str(codebook[-1]) + "\n") f.close() msg = "Export codebooks vector to " + filename + ".csv success" helper.write_log("dataset", '3', msg)
def train(root, device, model, epochs, bs, lr): print('start training ...........') train_loader, val_loader = get_loader(root=root, batch_size=bs, shuffle=True) optimizer = optim.SGD(model.parameters(), lr=lr, momentum=0.9, nesterov=True) scheduler = optim.lr_scheduler.CosineAnnealingWarmRestarts( optimizer, 50, 1) criterion = nn.CrossEntropyLoss() train_losses, train_acc = [], [] val_losses, val_acc = [], [] for epoch in range(epochs): scheduler.step(epoch) train_epoch_loss, train_epoch_acc = fit(epoch, model, optimizer, criterion, device, train_loader, phase='training') val_epoch_loss, val_epoch_acc = fit(epoch, model, optimizer, criterion, device, val_loader, phase='validation') print('-----------------------------------------') if epoch == 0 or val_epoch_acc >= np.max(val_acc): torch.save(model.state_dict(), 'output/weight.pth') train_losses.append(train_epoch_loss) train_acc.append(train_epoch_acc) val_losses.append(val_epoch_loss) val_acc.append(val_epoch_acc) write_figures('output', train_losses, val_losses, train_acc, val_acc) write_log('output', epoch, train_epoch_loss, val_epoch_loss, train_epoch_acc, val_epoch_acc) scheduler.step()
def train(): print('start training ...........') device = torch.device("cuda:0" if (torch.cuda.is_available()) else "cpu") model = Model().to(device) batch_size = 2 num_epochs = 100 learning_rate = 0.1 train_loader, val_loader = get_loader(batch_size=batch_size, shuffle=True) optimizer = optim.SGD(model.parameters(), lr=learning_rate, momentum=0.9, nesterov=True) scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min') criterion = DiceLoss(smooth=1.) train_losses, val_losses = [], [] for epoch in range(num_epochs): train_epoch_loss = fit(epoch, model, optimizer, criterion, device, train_loader, phase='training') val_epoch_loss = fit(epoch, model, optimizer, criterion, device, val_loader, phase='validation') print('-----------------------------------------') if epoch == 0 or val_epoch_loss <= np.min(val_losses): torch.save(model.state_dict(), 'output/weight.pth') train_losses.append(train_epoch_loss) val_losses.append(val_epoch_loss) write_figures('output', train_losses, val_losses) write_log('output', epoch, train_epoch_loss, val_epoch_loss) scheduler.step(val_epoch_loss)
def create_global_setting(self): """ Создается файл settings.ini. Файл находится в папке с программой. setup = true. Нужно зарегестрировать пользователя. setup = false. Пользователь зарегестрирован. :return: """ path = r"{0}\\settings.ini".format(getcwd()) try: file = open(path, "w") file.write("[global]\n") file.write("registration = true\n") file.write("version = {0}\n".format(version)) file.close() except IOError as e: helper.write_log("Ошибка I/O({0}): {1}".format(e.errno, e.strerror), "error") except: helper.write_log("непредвиденная ошибка: {0}".format(sys.exc_info()[0]), "error")
def import_codebooks(self, filename): dataset = list() with open(filename, 'r') as file: csv_reader = reader(file) n = 1 for row in csv_reader: if not row: continue dataset.append(row) n += 1 for i in range(len(dataset[0]) - 1): self.str_column_to_float(dataset, i) # convert class column to integers self.str_column_to_int(dataset, -1) self.n_codebooks = n self.codebooks = dataset msg = "Import codebooks vector from " + filename + " success" helper.write_log("dataset", '3', msg)
def get_events(days=8): today_bod = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) today_eod = today_bod + timedelta(1) hlp.write_log(datetime.now().strftime("### Getting data from Events API")) # Get last n days (including today) for d in range(days): start_dt = today_bod - timedelta(d) end_dt = today_eod - timedelta(d) start = str(int(datetime.timestamp(start_dt))) end = str(int(datetime.timestamp(end_dt))) target_url = config.fibaro_url + 'panels/event?' + 'from=' + start + "&" + "to=" + end r = call_fibaro_api(target_url) if r.status_code == 200: save_data(r.json(), start_dt.strftime('events_%Y-%m-%d-%H-%M-%S') + "_-_" + end_dt.strftime('%Y-%m-%d-%H-%M-%S')) hlp.write_log("Downloaded events from date: " + start_dt.strftime('%Y-%m-%d') + ": " + target_url) else: hlp.write_log("ERROR: error from API code: " + str(r.status_code)) time.sleep(0.1)
def save_data(json_data, filename_part): filename_with_path = os.path.join(config.data_directory, filename_part + ".json") temp_filename_with_path = filename_with_path + ".temp" bak_filename_with_path = os.path.join(config.data_backup_directory, filename_part + ".json" + datetime.now().strftime('.%Y-%m-%d-%H-%M-%S.bak')) exists = os.path.isfile(filename_with_path) if exists: hlp.write_json(json_data, temp_filename_with_path) if os.path.getsize(filename_with_path) == os.path.getsize(temp_filename_with_path): os.remove(temp_filename_with_path) hlp.write_log("File already existed and is the same. Keeping old file: " + filename_with_path, dtm_prefix=False) else: # files are different # backup old file os.rename(filename_with_path, bak_filename_with_path) # rename temporary file os.rename(temp_filename_with_path, filename_with_path) hlp.write_log("File already exists but it's different. Old file was backed up as: " + bak_filename_with_path, dtm_prefix=False) else: hlp.write_json(json_data, filename_with_path) hlp.write_log("File downloaded and stored as: " + filename_with_path, dtm_prefix=False)
def main(seed=25): seed_everything(25) device = torch.device('cuda:0') # arguments args = Args().parse() n_class = args.n_class img_path_train = args.img_path_train mask_path_train = args.mask_path_train img_path_val = args.img_path_val mask_path_val = args.mask_path_val model_path = os.path.join(args.model_path, args.task_name) # save model log_path = args.log_path output_path = args.output_path if not os.path.exists(model_path): os.makedirs(model_path) if not os.path.exists(log_path): os.makedirs(log_path) if not os.path.exists(output_path): os.makedirs(output_path) task_name = args.task_name print(task_name) ################################### evaluation = args.evaluation test = evaluation and False print("evaluation:", evaluation, "test:", test) ################################### print("preparing datasets and dataloaders......") batch_size = args.batch_size num_workers = args.num_workers config = args.config data_time = AverageMeter("DataTime", ':3.3f') batch_time = AverageMeter("BatchTime", ':3.3f') dataset_train = DoiDataset(img_path_train, config, train=True, root_mask=mask_path_train) dataloader_train = DataLoader(dataset_train, batch_size=batch_size, shuffle=True, num_workers=num_workers) dataset_val = DoiDataset(img_path_val, config, train=True, root_mask=mask_path_val) dataloader_val = DataLoader(dataset_val, batch_size=batch_size, shuffle=False, num_workers=num_workers) ################################### print("creating models......") model = DoiNet(n_class, config['min_descriptor'] + 6, 4) model = create_model_load_weights(model, evaluation=False, ckpt_path=args.ckpt_path) model.to(device) ################################### num_epochs = args.epochs learning_rate = args.lr optimizer = get_optimizer(model, learning_rate=learning_rate) scheduler = LR_Scheduler(args.scheduler, learning_rate, num_epochs, len(dataloader_train)) ################################## criterion_node = nn.CrossEntropyLoss() criterion_edge = nn.BCELoss() alpha = args.alpha writer = SummaryWriter(log_dir=log_path + task_name) f_log = open(log_path + task_name + ".log", 'w') ####################################### trainer = Trainer(criterion_node, criterion_edge, optimizer, n_class, device, alpha=alpha) evaluator = Evaluator(n_class, device) best_pred = 0.0 print("start training......") log = task_name + '\n' for k, v in args.__dict__.items(): log += str(k) + ' = ' + str(v) + '\n' print(log) f_log.write(log) f_log.flush() for epoch in range(num_epochs): optimizer.zero_grad() tbar = tqdm(dataloader_train) train_loss = 0 train_loss_edge = 0 train_loss_node = 0 start_time = time.time() for i_batch, sample in enumerate(tbar): data_time.update(time.time() - start_time) if evaluation: # evaluation pattern: no training break scheduler(optimizer, i_batch, epoch, best_pred) loss, loss_node, loss_edge = trainer.train(sample, model) train_loss += loss.item() train_loss_node += loss_node.item() train_loss_edge += loss_edge.item() train_scores_node, train_scores_edge = trainer.get_scores() batch_time.update(time.time() - start_time) start_time = time.time() if i_batch % 2 == 0: tbar.set_description( 'Train loss: %.4f (loss_node=%.4f loss_edge=%.4f); F1 node: %.4f F1 edge: %.4f; data time: %.2f; batch time: %.2f' % (train_loss / (i_batch + 1), train_loss_node / (i_batch + 1), train_loss_edge / (i_batch + 1), train_scores_node["macro_f1"], train_scores_edge["macro_f1"], data_time.avg, batch_time.avg)) trainer.reset_metrics() data_time.reset() batch_time.reset() if epoch % 1 == 0: with torch.no_grad(): model.eval() print("evaluating...") tbar = tqdm(dataloader_val) start_time = time.time() for i_batch, sample in enumerate(tbar): data_time.update(time.time() - start_time) pred_node, pred_edge = evaluator.eval(sample, model) val_scores_node, val_scores_edge = evaluator.get_scores() batch_time.update(time.time() - start_time) tbar.set_description( 'F1 node: %.4f F1 edge: %.4f; data time: %.2f; batch time: %.2f' % (val_scores_node["macro_f1"], val_scores_edge["macro_f1"], data_time.avg, batch_time.avg)) start_time = time.time() data_time.reset() batch_time.reset() val_scores_node, val_scores_node = evaluator.get_scores() evaluator.reset_metrics() best_pred = save_model(model, model_path, val_scores_node, val_scores_edge, alpha, task_name, epoch, best_pred) write_log(f_log, train_scores_node, train_scores_edge, val_scores_node, val_scores_edge, epoch, num_epochs) write_summaryWriter(writer, train_loss / len(dataloader_train), optimizer, train_scores_node, train_scores_edge, val_scores_node, val_scores_edge, epoch) f_log.close()
import fibaro_api_functions as api import helper as hlp hlp.write_log("##################### New process started") api.get_data("sections") api.get_data("rooms") api.get_data("scenes") api.get_data("devices") api.get_events() hlp.write_log("##################### Update process completed")
def __init__(self): #Проверка наличия фала settings.ini if not os.path.isfile('settings.ini'): helper.write_log('Нет файла settings.ini') #Запустить процедуру создания первоначального файла настроек self.create_settings()