from flask import render_template, request, send_from_directory, jsonify, url_for
from werkzeug.utils import secure_filename, redirect
from app import app
from keyword_cleaner import KeywordCleaner
from util.io import RedisClient
from util.log import get_logger
# This is the path to the upload directory
app.config['UPLOAD_FOLDER'] = os.path.join(
os.path.dirname(os.path.realpath(__file__)), 'upload')
# These are the extension that we are accepting to be uploaded
app.config['MAX_CONTENT_LENGTH'] = 1024 * 1024 * 1024 # Accept max 1GB file
logger = get_logger(__name__)
JOB_PREFIX = 'kwc'
@app.route('/')
def index():
return render_template('index.html')
@app.route('/keyword/batch-clean', methods=['POST'])
def keyword_batch_clean():
job_id = JOB_PREFIX + '-' + str(uuid4())
# Get form parameters
file_text = request.files.get('file_text')
import importlib
from util.log import get_logger
from config import config
logger = get_logger(
name='output',
level=config.get('log', {}).get('log_level', 'info'),
log_path=config.get('log', {}).get('log_path'))
def load_output(name, *args, **kwargs):
module = importlib.import_module('outputs.' + name)
cs = getattr(module, 'Output')
return cs(*args, **kwargs)
class OutputError(Exception):
pass
class BaseOutput(object):
def run(self):
raise NotImplementedError
import torch
import numpy as np
from torch.utils.data import DataLoader
from tqdm import tqdm
import H36M
import model.bilinear
from util import config
from util.log import get_logger
assert config.bilinear.comment is not None
logger, log_dir, comment = get_logger(comment=config.bilinear.comment)
data = DataLoader(
H36M.Dataset(
data_dir=config.bilinear.data_dir,
task=H36M.Task.Valid,
protocol=config.bilinear.protocol,
),
batch_size=config.bilinear.batch_size,
shuffle=True,
pin_memory=True,
num_workers=config.bilinear.num_workers,
)
bilinear, optimizer, step, train_epoch = model.bilinear.load(
device=config.bilinear.device,
parameter_dir='{log_dir}/parameter'.format(log_dir=log_dir),
)
bilinear.eval()
from Bio import SeqIO
from Bio.Blast.Applications import NcbiblastpCommandline
import Bio.PDB.Polypeptide as poly
import dotenv as de
import numpy as np
import tqdm
import sys
sys.path[-2] = '/home/users/sidhikab/lig_clash_score/src'
from util import log as log
from util import file as fi
project_root = os.path.abspath(os.path.join(__file__, '../../..'))
de.load_dotenv(os.path.join(project_root, '.env'))
logger = log.get_logger('sequence')
def find_similar(chain_sequences, blast_db, cutoff, num_alignments):
"""Find all other pdbs that have sequence identity greater than cutoff."""
if 'BLAST_BIN' not in os.environ:
raise RuntimeError('Need to set BLAST_BIN in .env to use blastp')
sim = set()
for chain, s in chain_sequences:
blastp_cline = NcbiblastpCommandline(db=blast_db,
outfmt="10 nident sacc",
num_alignments=num_alignments,
cmd=os.path.join(
os.environ['BLAST_BIN'],
# -*- coding: utf-8 -*-
# !/usr/bin/env python
import aimlcov.aimlbrain as aimlbrain
import analyze
import database
import car
import user
import os
import json
import re
import copy
import aimlcov.tuling as tuling
from macro import *
from util import log
logger = log.get_logger(__name__)
# the relationship between salebot, users and cars:
# a salebot is create by server
# a salebot like a host, can handle one user or multi users
# multi users like a chat room, no clearly design yet
# a salebot has one car to identify from db, the history find car can be saved in carlist attr
# each user has a carlist attr to remember what car has been watched before
# car contains the search para and search result, only search para be saved in carlist
class SaleBot(object):
def __init__(self,
userid=None,
carid=None,
aimlpath=os.path.split(os.path.realpath(__file__))[0] +
"/aimlcov/load_aiml.xml"):
aug_y_true = np.load(aug_features_path + '/labels.npy')
y_true = np.concatenate((y_true, aug_y_true))
y_pred = pipeline.predict(feature_matrix)
cnf_matrix = confusion_matrix(y_true, y_pred)
probabilities = pipeline.predict_proba(feature_matrix)
proba_cnf_matrix = probability_confusion_matrix(
y_true, y_pred, probabilities, FungusDataset.NUMBER_TO_FUNGUS)
results_path.mkdir(parents=True, exist_ok=True)
plot_all(results_path, mode, cnf_matrix, proba_cnf_matrix)
return accuracy_score(y_true, y_pred)
if __name__ == '__main__':
logger = get_logger('confusion_matrixes')
set_excepthook(logger)
args = parse_arguments()
config = load_config(args.config)
set_seed(config.seed)
train_results_path = get_results_path(config.results_path, args.model,
args.prefix, 'train')
test_results_path = get_results_path(config.results_path, args.model,
args.prefix, 'test')
train_features_path = get_results_path(config.results_path, args.features,
args.prefix, 'train')
test_features_path = get_results_path(config.results_path, args.features,
args.prefix, 'test')
model_path = get_results_path(config.results_path, args.features,
str(args.model) + '_' + str(args.prefix),
import torch
import torch.nn as nn
from torch.utils.data import DataLoader
from torchvision import transforms
from tqdm import tqdm
from tensorboardX import SummaryWriter
import H36M
import model.hourglass
from util import config
from util.visualize import colorize, overlap
from util.log import get_logger
assert config.hourglass.comment is not None
logger, log_dir, comment = get_logger(comment=config.hourglass.comment)
hourglass, optimizer, step, train_epoch = model.hourglass.load(
device=config.hourglass.device,
parameter_dir='{log_dir}/parameter'.format(log_dir=log_dir),
)
criterion = nn.MSELoss()
# Reset statistics of batch normalization
hourglass.reset_statistics()
hourglass.train()
train_loader = DataLoader(
H36M.Dataset(
data_dir=config.bilinear.data_dir,
task=H36M.Task.Train,
protocol=H36M.Protocol.GT,
def main(gpu, cfgs):
setproctitle.setproctitle('python')
##### config
global cfg
cfg = cfgs
cfg.local_rank = gpu
##### logger & summary writer
if cfg.local_rank == 0:
# logger
global logger
from util.log import get_logger
logger = get_logger(cfg)
# summary writer
global writer
writer = SummaryWriter(cfg.exp_path)
##### distributed training setting
if cfg.dist:
cfg.rank = cfg.node_rank * cfg.ngpu_per_node + gpu
print('[PID {}] rank: {} world_size: {}'.format(
os.getpid(), cfg.rank, cfg.world_size))
dist.init_process_group(backend='nccl',
init_method='tcp://127.0.0.1:%d' %
cfg.tcp_port,
world_size=cfg.world_size,
rank=cfg.rank)
torch.cuda.set_device(gpu)
assert cfg.batch_size % cfg.world_size == 0, 'Batch size should be matched with GPUS: (%d, %d)' % (
cfg.batch_size, cfg.world_size)
cfg.batch_size = cfg.batch_size // cfg.world_size
if cfg.local_rank == 0:
logger.info(cfg)
##### get model version and data version
exp_name = cfg.config.split('/')[-1][:-5]
model_name = exp_name.split('_')[0]
data_name = exp_name.split('_')[-1]
##### model
if cfg.local_rank == 0:
logger.info('=> creating model ...')
Network = model_class(cfg.model_name)
model = Network(cfg)
use_cuda = torch.cuda.is_available()
if cfg.local_rank == 0:
logger.info('cuda available: {}'.format(use_cuda))
assert use_cuda
model = model.to(gpu)
if cfg.dist:
if cfg.sync_bn:
model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(model).to(
gpu)
model = torch.nn.parallel.DistributedDataParallel(
model, device_ids=[gpu], find_unused_parameters=True)
if cfg.local_rank == 0:
# logger.info(model)
logger.info('#classifier parameters: {}'.format(
sum([x.nelement() for x in model.parameters()])))
##### optimizer
if cfg.optim == 'Adam':
optimizer = optim.Adam(filter(lambda p: p.requires_grad,
model.parameters()),
lr=cfg.lr)
elif cfg.optim == 'SGD':
optimizer = optim.SGD(filter(lambda p: p.requires_grad,
model.parameters()),
lr=cfg.lr,
momentum=cfg.momentum,
weight_decay=cfg.weight_decay)
##### model_fn (criterion)
model_fn = model_fn_decorator(cfg)
##### dataset
if cfg.dataset == 'scannetv2':
if data_name == 'scannet':
import data.scannetv2_inst
dataset = data.scannetv2_inst.ScannetDatast(cfg)
else:
print("Error: no data loader - " + data_name)
exit(0)
dataset.trainLoader()
dataset.valLoader()
if cfg.local_rank == 0:
logger.info('Training samples: {}'.format(
len(dataset.train_file_names)))
logger.info('Validation samples: {}'.format(
len(dataset.val_file_names)))
# f = utils.checkpoint_save(model, cfg.exp_path, cfg.config.split('/')[-1][:-5] + '_%d' % os.getpid(), 0, cfg.save_freq)
##### resume
start_epoch, f = utils.checkpoint_restore(
model,
cfg.exp_path,
cfg.config.split('/')[-1][:-5],
dist=cfg.dist,
f=cfg.pretrain,
gpu=gpu
) # resume from the latest epoch, or specify the epoch to restore
if cfg.local_rank == 0:
logger.info('Restore from {}'.format(f) if len(f) > 0 else
'Start from epoch {}'.format(start_epoch))
##### train and val
for epoch in range(start_epoch, cfg.epochs + 1):
if cfg.dist:
dataset.train_sampler.set_epoch(epoch)
train_epoch(dataset.train_data_loader, model, model_fn, optimizer,
epoch)
if cfg.validation:
if utils.is_multiple(epoch,
cfg.save_freq) or utils.is_power2(epoch):
if cfg.dist:
dataset.val_sampler.set_epoch(epoch)
eval_epoch(dataset.val_data_loader, model, model_fn, epoch)