def default_quota_set(): config = utils.load_settings(env) try: config["default-quota"] = validate_quota(request.values.get('default_quota')) utils.write_settings(config, env) except ValueError as e: return ("ERROR: %s" % str(e), 400) return "OK"
def smtp_relay_set(): from editconf import edit_conf config = utils.load_settings(env) newconf = request.form try: # Write on daemon settings config["SMTP_RELAY_ENABLED"] = (newconf.get("enabled") == "true") config["SMTP_RELAY_HOST"] = newconf.get("host") config["SMTP_RELAY_AUTH"] = (newconf.get("auth_enabled") == "true") config["SMTP_RELAY_USER"] = newconf.get("user") utils.write_settings(config, env) # Write on Postfix configs edit_conf("/etc/postfix/main.cf", [ "relayhost=" + (f"[{config['SMTP_RELAY_HOST']}]:587" if config["SMTP_RELAY_ENABLED"] else ""), "smtp_sasl_auth_enable=" + ("yes" if config["SMTP_RELAY_AUTH"] else "no"), "smtp_sasl_security_options=" + ("noanonymous" if config["SMTP_RELAY_AUTH"] else "anonymous"), "smtp_sasl_tls_security_options=" + ("noanonymous" if config["SMTP_RELAY_AUTH"] else "anonymous") ], delimiter_re=r"\s*=\s*", delimiter="=", comment_char="#") if config["SMTP_RELAY_AUTH"]: # Edit the sasl password with open("/etc/postfix/sasl_passwd", "w") as f: f.write( f"[{config['SMTP_RELAY_HOST']}]:587 {config['SMTP_RELAY_USER']}:{newconf.get('key')}\n" ) utils.shell("check_output", ["/usr/bin/chmod", "600", "/etc/postfix/sasl_passwd"], capture_stderr=True) utils.shell("check_output", ["/usr/sbin/postmap", "/etc/postfix/sasl_passwd"], capture_stderr=True) # Restart Postfix return utils.shell("check_output", ["/usr/bin/systemctl", "restart", "postfix"], capture_stderr=True) except Exception as e: return (str(e), 500)
def privacy_status_set(): config = utils.load_settings(env) config["privacy"] = (request.form.get('value') == "private") utils.write_settings(config, env) return "OK"
def main_worker(local_rank, args): best_acc1 = 0 best_acc1_index = 0 logger = None writer = None # os.environ['CUDA_VISIBLE_DEVICES'] = args.gpus args.outpath = args.outpath + '_' + args.arch if args.local_rank == 0: output_process(args.outpath) logger = get_logger(args.outpath, 'DistributedDataParallel_amp') writer = SummaryWriter(args.outpath) # distributed init args.nprocs = torch.cuda.device_count() dist.init_process_group(backend='nccl') torch.cuda.set_device(device=local_rank) if args.local_rank == 0: write_settings(args) logger.info(args) # create model if args.pretrained: ddp_print("=> using pre-trained model: {}".format(args.arch), logger, local_rank) model = models.__dict__[args.arch](pretrained=True) else: ddp_print('=> creating model: {}'.format(args.arch), logger, local_rank) model = models.__dict__[args.arch]() # sync BN if dist.is_available() and dist.is_initialized() and args.sync_batchnorm: ddp_print("=> using sync BN", logger, local_rank) model = nn.SyncBatchNorm.convert_sync_batchnorm(model) else: ddp_print("=> not use sync BN", logger, local_rank) model = model.cuda(device=local_rank) # When using a single GPU per process and per # DistributedDataParallel, we need to divide the batch size # ourselves based on the total number of GPUs we have args.batch_size = int(args.batch_size / args.nprocs) model = DDP(model, device_ids=[local_rank]) # define loss function (criterion) and optimizer criterion = nn.CrossEntropyLoss().cuda(device=local_rank) optimizer = optim.SGD(model.parameters(), args.lr, momentum=args.momentum, weight_decay=args.weight_decay) if args.lr_scheduler == 'steplr': lr_scheduler = MultiStepLR(optimizer, milestones=args.step, gamma=args.gamma) ddp_print('lr_scheduler: SGD MultiStepLR !!!', logger, local_rank) else: assert False, ddp_print( "invalid lr_scheduler={}".format(args.lr_scheduler), logger, local_rank) # dataloader traindir = os.path.join(args.data, 'train') valdir = os.path.join(args.data, 'val') normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) train_dataset = datasets.ImageFolder( traindir, transforms.Compose([ transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ])) train_sampler = DistributedSampler(train_dataset) train_loader = DataLoader(train_dataset, batch_size=args.batch_size, num_workers=args.workers, pin_memory=True, sampler=train_sampler) val_dataset = datasets.ImageFolder( valdir, transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize, ])) val_sampler = DistributedSampler(val_dataset) val_loader = DataLoader(val_dataset, batch_size=args.batch_size, num_workers=args.workers, pin_memory=True, sampler=val_sampler) if args.evaluate: validate(val_loader, model, criterion, args, logger, writer, epoch=-1, local_rank=local_rank) return 0 # amp scaler = GradScaler(enabled=args.use_amp) total_start = time.time() for epoch in range(args.start_epoch, args.epochs): # DDP train_sampler.set_epoch(epoch) val_sampler.set_epoch(epoch) epoch_start = time.time() lr_scheduler.step(epoch) # train for every epoch train(train_loader, model, criterion, optimizer, epoch, args, logger, writer, local_rank, scaler) # evaluate on validation set acc1 = validate(val_loader, model, criterion, args, logger, writer, epoch, local_rank) # remember best acc@1 and save checkpoint is_best = acc1 > best_acc1 if is_best: best_acc1_index = epoch best_acc1 = acc1 epoch_end = time.time() ddp_print( '||==> Epoch=[{:d}/{:d}]\tbest_acc1={:.4f}\tbest_acc1_index={}\ttime_cost={:.4f}s' .format(epoch, args.epochs, best_acc1, best_acc1_index, epoch_end - epoch_start), logger, local_rank) if args.local_rank == 0: # save model save_checkpoint( { 'epoch': epoch + 1, 'arch': args.arch, 'state_dict': model.module.state_dict(), 'best_acc1': best_acc1, }, is_best, args.outpath) total_end = time.time() ddp_print('||==> total_time_cost={:.4f}s'.format(total_end - total_start), logger, local_rank) if args.local_rank == 0: writer.close()
def main_worker(args): global best_acc1 global best_acc1_index # os.environ['CUDA_VISIBLE_DEVICES'] = args.gpus args.outpath = args.outpath + '_' + args.arch output_process(args.outpath) write_settings(args) logger = get_logger(args.outpath, 'DataParallel') writer = SummaryWriter(args.outpath) logger.info(args) # create model if args.pretrained: logger.info("=> using pre-trained model: {}".format(args.arch)) model = models.__dict__[args.arch](pretrained=True) else: logger.info('=> creating model: {}'.format(args.arch)) model = models.__dict__[args.arch]() model = nn.DataParallel(model).cuda() # define loss function (criterion) and optimizer criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(model.parameters(), args.lr, momentum=args.momentum, weight_decay=args.weight_decay) if args.lr_scheduler == 'steplr': lr_scheduler = MultiStepLR(optimizer, milestones=args.step, gamma=args.gamma) logger.info('lr_scheduler: SGD MultiStepLR !!!') else: assert False, logger.info("invalid lr_scheduler={}".format(args.lr_scheduler)) # logger.info('lr_scheduler={}'.format(lr_scheduler)) # dataloader traindir = os.path.join(args.data, 'train') valdir = os.path.join(args.data, 'val') normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) train_dataset = datasets.ImageFolder(traindir, transforms.Compose([ transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ])) train_loader = DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.workers, pin_memory=True) val_dataset = datasets.ImageFolder(valdir, transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize, ])) val_loader = DataLoader(val_dataset, batch_size=args.batch_size, shuffle=False, num_workers=args.workers, pin_memory=True) if args.evaluate: validate(val_loader, model, criterion, args, logger, writer, epoch=-1) return 0 total_start = time.time() for epoch in range(args.start_epoch, args.epochs): epoch_start = time.time() lr_scheduler.step(epoch) # train for every epoch train(train_loader, model, criterion, optimizer, epoch, args, logger, writer) # evaluate on validation set acc1 = validate(val_loader, model, criterion, args, logger, writer, epoch) # remember best acc@1 and save checkpoint is_best = acc1 > best_acc1 if is_best: best_acc1_index = epoch best_acc1 = acc1 epoch_end = time.time() logger.info('||==> Epoch=[{:d}/{:d}]\tbest_acc1={:.4f}\tbest_acc1_index={}\ttime_cost={:.4f}s' .format(epoch, args.epochs, best_acc1, best_acc1_index, epoch_end - epoch_start)) # save model save_checkpoint( { 'epoch': epoch + 1, 'arch': args.arch, 'state_dict': model.module.state_dict(), 'best_acc1': best_acc1, }, is_best, args.outpath) total_end = time.time() logger.info('||==> total_time_cost={:.4f}s'.format(total_end - total_start)) writer.close()
def sync(self): self.do_pull() self.do_push() utils.write_settings(self._conn, last_update=time.time())
def smtp_relay_set(): from editconf import edit_conf from os import chmod import re, socket, ssl config = utils.load_settings(env) newconf = request.form # Is DKIM configured? sel = newconf.get("dkim_selector") if sel is None or sel.strip() == "": config["SMTP_RELAY_DKIM_SELECTOR"] = None config["SMTP_RELAY_DKIM_RR"] = None elif re.fullmatch(r"[a-z\d\._]+", sel.strip()) is None: return ("The DKIM selector is invalid!", 400) elif sel.strip() == config.get("local_dkim_selector", "mail"): return ( f"The DKIM selector {sel.strip()} is already in use by the box!", 400) else: # DKIM selector looks good, try processing the RR rr = newconf.get("dkim_rr", "") if rr.strip() == "": return ("Cannot publish a selector with an empty key!", 400) components = {} for r in re.split(r"[;\s]+", rr): sp = re.split(r"\=", r) if len(sp) != 2: return ("DKIM public key RR is malformed!", 400) components[sp[0]] = sp[1] if not components.get("p"): return ("The DKIM public key doesn't exist!", 400) config["SMTP_RELAY_DKIM_SELECTOR"] = sel config["SMTP_RELAY_DKIM_RR"] = components relay_on = False implicit_tls = False if newconf.get("enabled") == "true": relay_on = True # Try negotiating TLS directly. We need to know this because we need to configure Postfix # to be aware of this detail. try: ctx = ssl.create_default_context() with socket.create_connection( (newconf.get("host"), int(newconf.get("port"))), 5) as sock: with ctx.wrap_socket(sock, server_hostname=newconf.get("host")): implicit_tls = True except ssl.SSLError as sle: # Couldn't connect via TLS, configure Postfix to send via STARTTLS print(sle.reason) except (socket.herror, socket.gaierror) as he: return ( f"Unable to resolve hostname (it probably is incorrect): {he.strerror}", 400) except socket.timeout: return ( "We couldn't connect to the server. Is it down or did you write the wrong port number?", 400) pw_file = "/etc/postfix/sasl_passwd" modify_password = True # Check that if the provided password is empty, that there was a password saved before if (newconf.get("key", "") == ""): if os.path.isfile(pw_file): modify_password = False else: return ( "Please provide a password/key (there is no existing password to retain).", 400) try: # Write on daemon settings config["SMTP_RELAY_ENABLED"] = relay_on config["SMTP_RELAY_HOST"] = newconf.get("host") config["SMTP_RELAY_PORT"] = int(newconf.get("port")) config["SMTP_RELAY_USER"] = newconf.get("user") config["SMTP_RELAY_AUTHORIZED_SERVERS"] = [ s.strip() for s in re.split(r"[, ]+", newconf.get("authorized_servers", []) or "") if s.strip() != "" ] utils.write_settings(config, env) # Write on Postfix configs edit_conf("/etc/postfix/main.cf", [ "relayhost=" + (f"[{config['SMTP_RELAY_HOST']}]:{config['SMTP_RELAY_PORT']}" if config["SMTP_RELAY_ENABLED"] else ""), f"smtp_tls_wrappermode={'yes' if implicit_tls else 'no'}" ], delimiter_re=r"\s*=\s*", delimiter="=", comment_char="#") # Edit the sasl password (still will edit the file, but keep the pw) with open(pw_file, "a+") as f: f.seek(0) pwm = re.match(r"\[.+\]\:[0-9]+\s.+\:(.*)", f.readline()) if (pwm is None or len(pwm.groups()) != 1) and not modify_password: # Well if this isn't a bruh moment return ( "Please provide a password/key (there is no existing password to retain).", 400) f.truncate(0) f.write( f"[{config['SMTP_RELAY_HOST']}]:{config['SMTP_RELAY_PORT']} {config['SMTP_RELAY_USER']}:{newconf.get('key') if modify_password else pwm[1]}\n" ) chmod(pw_file, 0o600) utils.shell("check_output", ["/usr/sbin/postmap", pw_file], capture_stderr=True) # Regenerate DNS (to apply whatever changes need to be made) from dns_update import do_dns_update do_dns_update(env) # Restart Postfix return utils.shell("check_output", ["/usr/sbin/postfix", "reload"], capture_stderr=True) except Exception as e: return (str(e), 400)