def run_cmd(self, *args):
"""Run help command"""
conf = get_config('help.json')
response = conf["help"]
if not self.option:
response = self.config["help"]
else:
get_help = get_config(f'{self.option}.json')
response = get_help["help"]
return "\n".join(response)
def __init__(self, event, user):
self.config = get_config('example.json')
self.parsed_args = SlackArgParse(self.config['valid_args'],
self.config['options'], event['text'])
self.args = self.parsed_args.args
self.option = self.parsed_args.option
self.response = self.run_cmd()
def __init__(self, args, option=None, team=None, player=None):
self.args = args
self.option = option
self.team = team
self.player = player
self.config = get_config('nhl_config.json')
self.emojis = self.config['emojis']
self.nhl = NHL()
def __init__(self, event, user):
self.config = get_config('help.json')
try:
self.option = event["text"].split()[1]
except IndexError:
self.option = False
self.option = False
self.user = event["user"]
def __init__(self, event, user):
self.text = event['text']
self.config = get_config('config.json')
self.parsed_args = SlackArgParse(self.config['valid_args'], self.config['options'], event['text'].lower())
self.args = self.parsed_args.args
self.option = self.parsed_args.option
self.league = self._get_league()
self.team_name = self._get_team_name()
self.response = self.run_cmd()
def get_general_page():
PATH2ROOT = Path('..')
PATH2CONFIG = Path(PATH2ROOT / 'configs')
CONFIG = get_config(PATH2CONFIG / 'config.yml')
tag_map = get_tag_map(PATH2CONFIG)
inv_tag_map = get_inv_tag_map(tag_map)
device = get_device(CONFIG)
model = get_model(PATH2ROOT, CONFIG, device, len(tag_map.keys()))
st.title('Named Entity Recognition using BERT')
st.subheader(
'This simple application provide a way to visualize token classification task (NER)💡'
)
text_key = st.selectbox('select some example or type your text',
list(TEXT_EXAMPLES.keys()))
raw_text = st.text_area("enter text here", value=TEXT_EXAMPLES[text_key])
raw_text_split = raw_text.split()
test_dataset = NamedEntityRecognitionDataset(
texts=[raw_text_split],
tags=[[0] * len(raw_text_split)],
tokenizer=CONFIG['model']['model_name'],
max_seq_len=CONFIG['model']['max_seq_length'],
lazy_mode=False,
)
if st.button('Submit🔥'):
if raw_text == '':
st.warning('Please, enter some text...')
return
elif len(raw_text_split) > CONFIG['model']['max_seq_length']:
st.warning(
f"Text should be shorter than {CONFIG['model']['max_seq_length']} tokens..."
)
return
ann_text = get_prediction(model, test_dataset, device)
annotated_text(*[
text +
" " if class_ == tag_map['O'][0] else (text + " ",
inv_tag_map[class_][0],
inv_tag_map[class_][1])
for text, class_ in ann_text
])
ann_text = np.array(ann_text)
db_insert(
raw_text,
json.dumps(ann_text[:, 0].tolist(), ensure_ascii=False),
json.dumps(CONFIG),
json.dumps(ann_text[:, 1].tolist(), ensure_ascii=False),
)
def __init__(self, token):
"""
Initialize the Slack object given the provided bot token
:param token:
:param config:
"""
self.config = get_config('slack.json')
self.client = slackclient.SlackClient(token)
self.commands = self.load_commands('/jockbot/commands/')
def __init__(self, api_version="1.2"):
self.api_key = os.environ.get('MYSPORTSFEEDS_API_KEY')
self.version = api_version
self.password = os.environ.get('MYSPORTSFEEDS_PASSWORD')
self.date = datetime.datetime.now()
self.base_url = f"https://api.mysportsfeeds.com/v{self.version}/pull/nfl/"
self.session = requests.session()
self.league_schedule = self.get_schedule()
self.upcoming_games = self.get_games_by_week()
self.config = get_config('nfl_config.json')
self.league_game_results = []
self.played_games = []
self.unplayed_games = []
self.league_played_games = []
self.league_unplayed_games = []
self.loop = asyncio.new_event_loop()
self.loop.run_until_complete(self.parse_league_games())
self.loop.run_until_complete(self.gather_league_data())
def get_about_page():
st.title('About')
PATH2ROOT = Path('..')
PATH2CONFIG = Path(PATH2ROOT / 'configs')
CONFIG = get_config(PATH2CONFIG / 'config.yml')
st.header('Model parameters')
st.json(CONFIG)
st.header('Loss function plot')
st.markdown(
''
)
st.header('Accuracy plot')
st.markdown(
''
)
from contextlib import closing
from mysql.connector import OperationalError
from utils.helpers import get_config, get_connection
if __name__ == '__main__':
config = get_config()
with closing(get_connection(**config)) as conn:
print('Connection successful')
c = conn.cursor()
update = """
update customer as c, address as a
set c.email = %s, a.phone = %s
where c.address_id = a.address_id and c.customer_id = 100;
"""
email = '*****@*****.**'
phone = '949-555-1234'
c.execute(update, (email, phone))
conn.commit()
def main(data_path='/data/SN6_buildings/train/AOI_11_Rotterdam/',
config_path='/project/configs/senet154_gcc_fold1.py',
gpu='0'):
os.environ["CUDA_VISIBLE_DEVICES"] = gpu
config = get_config(config_path)
model_name = config['model_name']
fold_number = config['fold_number']
alias = config['alias']
log_path = osp.join(config['logs_path'],
alias + str(fold_number) + '_' + model_name)
device = torch.device(config['device'])
weights = config['weights']
loss_name = config['loss']
optimizer_name = config['optimizer']
lr = config['lr']
decay = config['decay']
momentum = config['momentum']
epochs = config['epochs']
fp16 = config['fp16']
n_classes = config['n_classes']
input_channels = config['input_channels']
main_metric = config['main_metric']
best_models_count = config['best_models_count']
minimize_metric = config['minimize_metric']
min_delta = config['min_delta']
train_images = data_path
data_type = config['data_type']
masks_data_path = config['masks_data_path']
folds_file = config['folds_file']
train_augs = config['train_augs']
preprocessing_fn = config['preprocessing_fn']
limit_files = config['limit_files']
batch_size = config['batch_size']
shuffle = config['shuffle']
num_workers = config['num_workers']
valid_augs = config['valid_augs']
val_batch_size = config['val_batch_size']
multiplier = config['multiplier']
train_dataset = SemSegDataset(images_dir=train_images,
data_type=data_type,
masks_dir=masks_data_path,
mode='train',
n_classes=n_classes,
folds_file=folds_file,
fold_number=fold_number,
augmentation=train_augs,
preprocessing=preprocessing_fn,
limit_files=limit_files,
multiplier=multiplier)
train_loader = DataLoader(dataset=train_dataset,
batch_size=batch_size,
shuffle=shuffle,
num_workers=num_workers)
valid_dataset = SemSegDataset(images_dir=train_images,
data_type=data_type,
mode='valid',
folds_file=folds_file,
n_classes=n_classes,
fold_number=fold_number,
augmentation=valid_augs,
preprocessing=preprocessing_fn,
limit_files=limit_files)
valid_loader = DataLoader(dataset=valid_dataset,
batch_size=val_batch_size,
shuffle=False,
num_workers=num_workers)
model = make_model(model_name=model_name,
weights=weights,
n_classes=n_classes,
input_channels=input_channels).to(device)
loss = get_loss(loss_name=loss_name)
optimizer = get_optimizer(optimizer_name=optimizer_name,
model=model,
lr=lr,
momentum=momentum,
decay=decay)
if config['scheduler'] == 'reduce_on_plateau':
print('reduce lr')
alpha = config['alpha']
patience = config['patience']
threshold = config['thershold']
min_lr = config['min_lr']
mode = config['scheduler_mode']
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
optimizer=optimizer,
factor=alpha,
verbose=True,
patience=patience,
mode=mode,
threshold=threshold,
min_lr=min_lr)
elif config['scheduler'] == 'steps':
print('steps lr')
steps = config['steps']
step_gamma = config['step_gamma']
scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer=optimizer,
milestones=steps,
gamma=step_gamma)
else:
scheduler = None
callbacks = []
dice_callback = DiceCallback()
callbacks.append(dice_callback)
callbacks.append(CheckpointCallback(save_n_best=best_models_count))
callbacks.append(
EarlyStoppingCallback(patience=config['early_stopping'],
metric=main_metric,
minimize=minimize_metric,
min_delta=min_delta))
runner = SupervisedRunner(device=device)
loaders = {'train': train_loader, 'valid': valid_loader}
runner.train(model=model,
criterion=loss,
optimizer=optimizer,
loaders=loaders,
scheduler=scheduler,
callbacks=callbacks,
logdir=log_path,
num_epochs=epochs,
verbose=True,
main_metric=main_metric,
minimize_metric=minimize_metric,
fp16=fp16)
def main(config_path='/project/configs/senet154_gcc_fold1.py',
test_images='/data/SN6_buildings/test_public/AOI_11_Rotterdam/',
test_predict_result='/wdata/folds_predicts/',
batch_size=1,
workers=1,
gpu='1'):
with torch.no_grad():
config = get_config(config_path)
model_name = config['model_name']
weights_path = config['load_from']
device = config['device']
val_batch_size = batch_size
input_channels = config['input_channels']
original_size = config['original_size']
cropper = albu.Compose(
[albu.CenterCrop(original_size[0], original_size[1], p=1.0)])
n_classes = config['n_classes']
preprocessing_fn = config['preprocessing_fn']
valid_augs = config['valid_augs']
limit_files = config['limit_files']
num_workers = workers
os.environ["CUDA_VISIBLE_DEVICES"] = gpu
if not os.path.exists(test_predict_result):
os.mkdir(test_predict_result)
fold_name = weights_path.split('/')[-3]
folder_to_save = os.path.join(test_predict_result, fold_name)
if os.path.exists(folder_to_save):
shutil.rmtree(folder_to_save)
os.mkdir(folder_to_save)
test_dataset = TestSemSegDataset(images_dir=os.path.join(
test_images, 'SAR-Intensity'),
preprocessing=preprocessing_fn,
augmentation=valid_augs,
limit_files=limit_files)
test_loader = DataLoader(dataset=test_dataset,
batch_size=val_batch_size,
shuffle=False,
num_workers=num_workers)
print('Loading {}'.format(weights_path))
model = make_model(model_name=model_name,
weights=None,
n_classes=n_classes,
input_channels=input_channels).to(device)
model.load_state_dict(torch.load(weights_path)['model_state_dict'])
model.eval()
model = tta.TTAWrapper(model, flip_image2mask)
model = torch.nn.DataParallel(model).cuda()
file_names = sorted(test_dataset.ids)
for batch_i, test_batch in enumerate(tqdm(test_loader)):
runner_out = model(test_batch.cuda())
image_pred = runner_out
image_pred = image_pred.cpu().detach().numpy()
names = file_names[batch_i * val_batch_size:(batch_i + 1) *
val_batch_size]
for i in range(len(names)):
file_name = os.path.join(folder_to_save,
names[i].split('.')[0] + '.png')
data = image_pred[i, ...]
data = np.moveaxis(data, 0, -1)
sample = cropper(image=data)
data = sample['image']
data = (data * 255).astype(np.uint8)
cv2.imwrite(file_name, data)
import joblib
import torch
from catalyst.utils import prepare_cudnn, set_global_seed
from sklearn import model_selection
from transformers import AdamW, get_linear_schedule_with_warmup
from datasets import NamedEntityRecognitionDataset
from models import NamedEntityRecognitionBertModel
from models.loss_fn import FocalLossCustom
from utils.callbacks import AccuracyCallbackCustom
from utils.helpers import get_config, remove_dir
from utils.runners import CustomRunner
PATH2ROOT = Path('..')
PATH2CONFIG = Path(PATH2ROOT / 'configs')
CONFIG = get_config(PATH2CONFIG / 'config.yml')
PATH2CORPUS = Path(PATH2ROOT / CONFIG['data']['path_to_corpus_folder'])
MODEL_NAME = CONFIG['model']['model_name']
if __name__ == "__main__":
data = joblib.load(PATH2ROOT / CONFIG['data']['path_to_preproc_data'])
texts = data['text'].values.tolist()
tags = data['tags'].values.tolist()
with open(PATH2CONFIG / 'target_mapper.json', 'r') as file:
tag_map = json.load(file)
tag_map = {key: val[0] for key, val in tag_map.items()}
for i in range(len(tags)):
for j in range(len(tags[i])):