def create_app():
"""
Creates the flask app and sets up logging.
"""
eventservice = Flask(__name__)
setup_logging()
return eventservice
def create_app():
"""
Creates the flask app and sets up logging.
:return: the flask app.
"""
calendarservice = Flask(__name__)
setup_logging()
return calendarservice
def create_app():
"""
Creates the flask app and sets up logging.
:return: the flask app.
"""
accountservice = Flask(__name__)
setup_logging()
return accountservice
def __init__(self):
# Parse initial experiment arguments
initial_arguments = ExperimentArguments(sections=('experiment', ),
use_all_cli_args=False)
initial_arguments.add_class_arguments(Experiment)
initial_arguments.get_arguments()
self.is_master = initial_arguments.is_master
self.initialize_backend(initial_arguments.backend.title())
self.ModelClass = getattr(model_module,
initial_arguments.model_class_name)
self.ReaderClass = getattr(reader_module,
initial_arguments.reader_class_name)
self.BudgetDecoderClass = getattr(
src.hpbandster.budget_decoder,
initial_arguments.budget_decoder_class_name)
self.TrainerClass = TrainerClass
# Populate experiment arguments with arguments from specific classes
self.experiment_arguments = ExperimentArguments(use_all_cli_args=True)
self.experiment_arguments.add_class_arguments(Experiment)
self.experiment_arguments.add_class_arguments(self.ModelClass)
self.experiment_arguments.add_class_arguments(self.ReaderClass)
self.experiment_arguments.add_class_arguments(self.TrainerClass)
self.experiment_arguments.add_class_arguments(TrainManager)
self.experiment_arguments.add_class_arguments(BayesianOptimizer)
self.experiment_arguments.add_class_arguments(ConfigGenerator)
self.experiment_arguments.get_arguments()
verbose = initial_arguments.verbose
setup_logging(self.experiment_arguments.working_dir,
logging.DEBUG if verbose else logging.INFO)
self.train_manager = TrainManager(
ModelClass=self.ModelClass,
ReaderClass=self.ReaderClass,
TrainerClass=self.TrainerClass,
**self.experiment_arguments.get_arguments())
self.budget_decoder = self.BudgetDecoderClass(
**self.experiment_arguments.get_arguments())
# DataReader can't be instantiated properly before the model is created
def context_size(**kwargs):
return 1
@staticmethod
def input_size(**kwargs):
return 3
@staticmethod
def output_size(**kwargs):
return 4
if __name__ == '__main__':
from src.utils import setup_logging
setup_logging('/tmp', logging.DEBUG)
data_reader = BBCIDataReader(data_path='/home/schirrmr/data/',
readers_count=1,
batch_size=64,
validation_batch_size=0,
sequence_size=1125,
validation_sequence_size=4500,
balanced=1,
random_mode=2,
continuous=1,
limit_examples=0,
limit_duration=0,
forget_state=1,
train_on_full=0,
cv_n=3,
cv_k=2,
#! /usr/bin/env python # -*- coding: utf-8 -*- # vim:fenc=utf-8 """ 平时都是使用此文件导入配置，但是有些特殊的需要和开发者机器相关的，需要在local_configs中书写 """ from .local_configs import * from src.utils import setup_logging setup_logging()
def main(args):
# Logging
LOGGER = logging.getLogger(__name__)
exp_dir = os.path.join("experiments", f"{args.backbone}_v{args.version}")
log_file = os.path.join(exp_dir, "log.log")
loss_file = os.path.join(exp_dir, "results_loss.npy")
confusion_file = os.path.join(exp_dir, "results_confusion.npy")
os.makedirs(exp_dir, exist_ok=True)
setup_logging(log_path=log_file, log_level=args.log_level, logger=LOGGER)
args_file = os.path.join(exp_dir, "args.log")
with open(args_file, "w") as f:
args_logger = ""
for k, v in args.__dict__.items():
args_logger = "\n".join([args_logger, f"{k}: {v}"])
f.write(args_logger)
# Initialize datasets and loaders.
LOGGER.info("Data Processing...")
df = generate_meta(args.data_dir)
train_ids, valid_ids = stratified_split(df)
n_classes = df["target"].max() + 1
if len(args.loss_weight) != n_classes:
raise ValueError(
f"Length of argument `loss-weight` should be {n_classes}, \
got {len(args.loss_weight)} instead.")
# Build model and get pretrained weight by pretrainedmodels
model = HAMNet(n_classes, model_name=args.backbone)
model = model.to(device)
train_df = df.loc[df["image_id"].isin(train_ids)]
train_df = over_sample(train_df, args.imbalanced_weight, frac=1.0)
valid_df = df.loc[df["image_id"].isin(valid_ids)]
train_dataset = HAMDataset(train_df,
build_preprocess(model.mean, model.std),
build_train_transform())
valid_dataset = HAMDataset(valid_df,
build_preprocess(model.mean, model.std),
build_test_transform())
train_loader = DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=10)
valid_loader = DataLoader(valid_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=10)
optimizer = optim.Adam(model.parameters(), lr=args.lr)
scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=300)
weights = torch.tensor(args.loss_weight).to(device)
criterion = FocalLoss(weight=weights, gamma=2).to(device)
start_epoch = 1
if args.load_version:
# Find the weight corresponding to the best score
weight_folder = os.path.join("experiments",
f"{args.backbone}_v{args.load_version}")
weights = [w for w in glob(f"{weight_folder}/*.ckpt")]
best_idx = np.argmax([w.split("/")[-1] for w in weights])
best_weight = weights[best_idx]
ckpt = load_checkpoint(path=best_weight,
model=model,
optimizer=optimizer,
epoch=True)
model, optimizer, start_epoch = (ckpt["model"], ckpt["optimizer"],
ckpt["epoch"] + 1)
model = model.to(device)
best_map = 0
epochs, train_losses, valid_losses = [], [], []
for epoch in range(start_epoch, start_epoch + args.num_epochs):
# Training
LOGGER.info(f"Epoch: {epoch}")
model.train()
n_batches = len(train_loader.dataset) // args.batch_size + 1
train_loss = AverageMeter()
train_acc = AverageMeter()
train_f = AverageMeter()
for batch_idx, (inputs, targets, _) in enumerate(train_loader):
inputs, targets = (inputs.to(device),
targets.type(torch.LongTensor).to(device))
outputs = model(inputs)
loss = criterion(outputs, targets)
loss.backward()
# Gradient accumulation for larger batch size
if (batch_idx + 1) % args.acc_steps == 0:
optimizer.step()
scheduler.step()
optimizer.zero_grad()
_, predicted = outputs.max(dim=1)
# Update Metrics
from src.metrics import avg_precision, avg_fscore
train_loss.update(loss.item())
train_acc.update(
predicted.eq(targets).sum().item() / targets.size(0))
train_f.update(
avg_fscore(targets.cpu().detach().numpy(),
predicted.cpu().detach().numpy()))
if batch_idx % 10 == 9:
LOGGER.info(
STATUS_MSG_T.format(batch_idx + 1, n_batches,
train_loss.avg, train_acc.avg,
train_f.avg))
# Validation
val_ids, val_labels, val_preds = [], [], []
model.eval()
valid_loss = AverageMeter()
valid_acc = AverageMeter()
valid_f = AverageMeter()
with torch.no_grad():
for batch_idx, (inputs, targets,
img_id) in enumerate(valid_loader):
inputs, targets = (inputs.to(device),
targets.type(torch.LongTensor).to(device))
# Apply Test Time Augmentation
tta = TestTimeAugment(model, times=args.tta)
outputs = tta.predict(inputs)
loss = criterion(outputs, targets)
_, predicted = outputs.max(dim=1)
valid_loss.update(loss.item())
valid_acc.update(
predicted.eq(targets).sum().item() / targets.size(0))
valid_f.update(
avg_precision(targets.cpu().numpy(),
predicted.cpu().numpy()))
val_ids.extend(img_id)
val_labels.extend(targets.cpu().numpy())
val_preds.extend(np.squeeze(predicted.cpu().numpy().T))
LOGGER.info(
STATUS_MSG_V.format(epoch, args.num_epochs + start_epoch,
valid_loss.avg, valid_acc.avg, valid_f.avg))
# Save checkpoint.
if valid_f.avg > best_map:
LOGGER.info("Saving..")
output_file_name = os.path.join(
exp_dir, f"checkpoint_{valid_f.avg:.3f}.ckpt")
save_checkpoint(path=output_file_name,
model=model,
epoch=epoch,
optimizer=optimizer)
best_map = valid_f.avg
epochs.append(epoch)
train_losses.append(train_loss.avg)
valid_losses.append(valid_loss.avg)
create_loss_plot(exp_dir, epochs, train_losses, valid_losses)
np.save(loss_file, [train_losses, valid_losses])
np.save(confusion_file, [val_ids, val_labels, val_preds])
confusion_mtx = confusion_matrix(val_labels, val_preds)
plot_labels = ["akiec", "bcc", "bkl", "df", "mel", "nv", "vasc"]
create_confusion_matrix(exp_dir,
confusion_mtx,
plot_labels,
normalize=True)
import asyncio
import aiohttp
from src.utils import setup_logging
logger = setup_logging(streamlevel="INFO", filelevel=None)
class BaseAsyncCrawler:
HEADER = {
"User-Agent": (
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36"
" (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36"
),
"Connection": "keep-alive",
"Accept": "text/html",
"Accept-Charset": "utf-8",
}
TIMEOUT = aiohttp.ClientTimeout(total=60)
MAX_CONCURRENCY = 30
def urls(self):
raise NotImplementedError
async def download(self, session, url):
raise NotImplementedError
if self._thread is None:
self._thread = Thread(target=self._run_processing)
self._thread.daemon = True
self._thread.start()
def stop(self):
self._stopped = True
def wait(self):
if self._thread is not None:
self._thread.join()
self._thread = None
if __name__ == "__main__":
setup_logging(settings.log_level)
auth = tweepy.OAuthHandler(settings.consumer_key, settings.consumer_secret)
auth.set_access_token(settings.access_token, settings.access_token_secret)
twitter_api = tweepy.API(auth,
wait_on_rate_limit=True,
wait_on_rate_limit_notify=True)
twitter_api.verify_credentials()
logger.info("Credentials verified")
predictor_params = {
"feature_checkpoint_path": settings.feature_checkpoint_path,
"feature_config_path": settings.feature_config_path,
"caption_checkpoint_path": settings.caption_checkpoint_path,
"caption_config_path": settings.caption_config_path,
"beam_size": settings.caption_beam_size,
pickle.dump(vocab, f)
early_stopping = EarlyStopping(monitor="val_loss",
mode="min",
verbose=True,
patience=3)
lr_logger = LearningRateMonitor()
# Loss and optimizer
trainer = pl.Trainer(
gpus=1,
max_epochs=max_epochs,
progress_bar_refresh_rate=20,
callbacks=[lr_logger, early_stopping],
)
# Auto log all MLflow entities
mlflow.pytorch.autolog(log_models=False)
with mlflow.start_run() as run:
logger.info(f"run_id: {run.info.run_id}")
mlflow.log_artifact(VOCAB_DUMP_PATH, artifact_path="model/data")
trainer.fit(model, train_dataloader, test_dataloader)
log_model(model)
if __name__ == "__main__":
logger = setup_logging()
args = get_parser().parse_args()
main(args.n_grams, args.batch_size, args.embed_dim, args.max_epochs)