def load_configuration():
global dropbox_client, local_cloud_path, upload_prefix, enable_upload, download_prefix, enable_download, \
thin_mode_byte_length, tmp_path, sync_interval_in_sec
file_handle = open(CONFIGURATION_FILE, 'r')
config_data = file_handle.read()
file_handle.close()
configs = json.loads(config_data)
init_logger(configs["server_configuration"]["log_path"])
local_cloud_path = configs["server_configuration"]["local_cloud_path"]
upload_prefix = configs["server_configuration"]["upload_filename_magic"]
download_prefix = configs["server_configuration"][
"download_filename_magic"]
enable_download = json.loads(
configs["server_configuration"]["enable_download"].lower())
enable_upload = json.loads(
configs["server_configuration"]["enable_upload"].lower())
thin_mode_byte_length = int(
configs["server_configuration"]["bytes_length_in_thin_mode"])
tmp_path = configs["server_configuration"]["tmp_path"]
sync_interval_in_sec = configs["server_configuration"][
"sync_interval_in_sec"]
if configs["cloud_credential"]["dropbox"][
"access_token"] is not DEFAULT_ACCESS_TOCKEN:
dropbox_client = DropboxClient(
configs["cloud_credential"]["dropbox"]["access_token"])
logger.info("Finish loading configuration")
def init_performance_generation(experiment_id: str, artifacts:list) -> Experiment:
'''Initalizes and creates a neptune experiment.'''
cache_dir = './artifacts'
for path in [cache_dir, 'source', 'trill_source']:
if os.path.exists(path):
shutil.rmtree(path)
os.mkdir(cache_dir)
# load the trill model and params
exp:Experiment = get_experiment_by_id('THESIS-75')
exp.download_artifact('trill_params.pickle', cache_dir)
exp.download_artifact('trill_best.pth', cache_dir)
exp.download_sources()
with zipfile.ZipFile('source.zip', 'r') as zip_ref:
zip_ref.extractall('./')
os.rename('./source', './trill_source')
init_logger()
exp:Experiment = get_experiment_by_id(experiment_id)
for art in artifacts:
exp.download_artifact(art, f'{cache_dir}')
exp.download_sources()
with zipfile.ZipFile('source.zip', 'r') as zip_ref:
zip_ref.extractall('./')
return exp
def init_training_job(is_dev:bool, exp_name:str, exp_description:str, hyper_params:Params, job_params: ModelJobParams, upload_files:list, tags:list = None) -> Experiment:
'''Initalizes and creates a neptune experiment. '''
hyper_params_dict = asdict(hyper_params)
job_params_dict = asdict(job_params)
hyper_params_dict.update(job_params_dict)
hyper_params_dict['device'] = f'cuda:{hyper_params.device_num}'
print(json.dumps(hyper_params_dict, indent=4))
if os.path.exists('./artifacts'):
shutil.rmtree('./artifacts')
logger = init_logger()
exp_tags = [f'{"dev" if is_dev else "full"}'] + tags
exp:Experiment = init_experiment(
exp_name=exp_name,
exp_description=exp_description,
tags=exp_tags,
params=hyper_params_dict,
upload_files=upload_files,
logger=logger
)
logger.info('Starting experiment')
log_neptune_timeline('Starting experiment', exp)
return exp
def train():
init_logger(tags=["debug"])
num_items = _get_num_items()
model = LSTMModel(num_items)
dm = BrightkiteDataModule()
checkpoint_callback = DVCLiveCompatibleModelCheckpoint(
dirpath="artifacts",
filename="lstm",
save_top_k=-1,
verbose=True,
)
dvclive_next_step_callback = DVCLiveNextStepCallback()
trainer = pl.Trainer(checkpoint_callback=checkpoint_callback,
deterministic=True,
logger=False,
max_epochs=params.lstm.optimizer.epochs,
gpus=-1,
callbacks=[dvclive_next_step_callback])
trainer.fit(model, dm)
def init_legacy_training_job(is_dev:bool, exp_name:str, exp_description:str, params:dict, tags:list=None):
'''Initalizes and creates a neptune experiment. '''
if os.path.exists('./artifacts'):
shutil.rmtree('./artifacts')
logger = init_logger()
exp_tags = [f'{"dev" if is_dev else "full"}'] + tags
upload_files = [f'{SRC_DIR}/old/nnModel.py', f'{SRC_DIR}/old/model_run.py']
exp:Experiment = init_experiment(
exp_name=exp_name,
exp_description=exp_description,
params=params,
tags=exp_tags,
upload_files=upload_files,
logger=logger
)
logger.info('Starting experiment')
log_neptune_timeline('Starting experiment', exp)
return exp
# To add a new cell, type '# %%'
# To add a new markdown cell, type '# %% [markdown]'
# %%
from IPython import get_ipython
# %%
get_ipython().run_line_magic('load_ext', 'autoreload')
get_ipython().run_line_magic('autoreload', '2')
# %%
from src.logger import init_logger
# %%
from src.constants import CACHE_DATA_DIR
from src.data.data_reader.read_featurized_cache import read_featurized
from src.experiments.training.BL.HANBaseline_training import run_han_bl_job
# %%
logger_path = 'logs/run_dev_data_0.1.log'
init_logger(logger_path)
# %%
path = f'{CACHE_DATA_DIR}/train/training_data_development.pickle'
dev_data = read_featurized(path)
# %%
num_epochs = 5
run_han_bl_job(dev_data, num_epochs, version=0.3, is_dev=True)
# %%
import logging
from wxpy import Bot, embed, Group, TEXT, SHARING, Friend
from src import config
from src.database import DataBase
from src.callback import qr_callback, logout_callback, login_callback
from src.mail import Mail
from src.logger import init_logger
from src.command import master_command, administrator_command
init_logger()
DataBase.init_database()
Mail.init_mail(user=config.EMAIL_USER,
password=config.EMAIL_PASSWORD,
host=config.EMAIL_HOST,
port=config.EMAIL_PORT)
logger = logging.getLogger(__name__)
if config.LOGIN_TYPE == config.MAIL:
bot = Bot(cache_path=True,
qr_callback=qr_callback,
login_callback=login_callback,
logout_callback=logout_callback)
else:
bot = Bot(cache_path=True,
login_callback=login_callback,
logout_callback=logout_callback)
def __init__(self, verbose=True):
"""进行初始化"""
self.verbose = verbose
self.scheduler = BlockingScheduler(timezone=config.timezone)
self.logger = init_logger()
]
app = web.Application()
for route in ROUTES:
app.router.add_route(method=route[0],
path=route[1],
name=route[2],
handler=route[3])
static_path = config.get("static_path")
if not os.path.exists(static_path):
os.mkdir(static_path)
app['static_root_url'] = config.get('static_path')
app.router.add_static(prefix='/static', path='static', name='static')
app.router.add_static(prefix='/', path='static', name='index')
app.on_cleanup.append(on_shutdown)
app['websockets'] = []
app['logger'] = init_logger()
app['users'] = dict()
app['logger'].info("Start")
try:
web.run_app(app, port=config.getint('port'))
except OSError as e:
app['logger'].error(f"Port: {config.getint('port')} is already in use.")
app['logger'].info("Server closing")
def __init__(self, verbose=True):
self.session = requests.Session()
self.logger = init_logger()
self.is_login = False
self.verbose = verbose
def main():
parser = get_parser()
params = parser.parse_args()
if not os.path.exists(params.exp_path):
os.makedirs(params.exp_path)
# some short-cut
src_lg = params.src_lang
tgt_lg = params.tgt_lang
log_file = os.path.join(params.exp_path, f'ramen_{src_lg}-{tgt_lg}.log')
logger = init_logger(log_file)
logger.info(params)
pretrained_model, adaptor, mask_index = get_model(params)
src_model = pretrained_model.from_pretrained(params.src_pretrained_path)
tgt_model = pretrained_model.from_pretrained(params.tgt_pretrained_path)
model = adaptor(src_model, tgt_model)
model = model.cuda()
optimizer = AdamInverseSquareRootWithWarmup(model.parameters(),
lr=params.lr,
warmup_updates=4000)
model, optimizer = amp.initialize(model,
optimizer,
opt_level=params.opt_level)
data = DataIterator(params)
train_loader_fr = data.get_iter(tgt_lg, 'train')
train_loader_en = data.get_iter(src_lg, 'train')
valid_loader_fr = data.get_iter(tgt_lg, 'valid')
valid_loader_en = data.get_iter(src_lg, 'valid')
def evaluate(lang, loader):
# useful for evaluation
model.eval()
losses = []
for x in loader:
x, y = mask_input(x.squeeze(0), params.word_pred, mask_index)
with torch.no_grad():
loss = model(lang, x, masked_lm_labels=y)
losses.append(loss.item())
model.train()
return sum(losses) / len(losses)
def step(lg, x, update=True):
x, y = mask_input(x.squeeze(0), params.word_pred, mask_index)
loss = model(lg, x, masked_lm_labels=y)
with amp.scale_loss(loss, optimizer) as scaled_loss:
scaled_loss.backward()
if update:
torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), 5)
optimizer.step()
optimizer.zero_grad()
return loss.item()
n_iter = 0
n_epoch = 0
start_time = time.time()
best_valid_nll = 1e8
model_prefix = 'roberta' if 'roberta' in params.src_pretrained_path else 'bert'
while True:
for batch_en, batch_fr in zip(train_loader_en, train_loader_fr):
n_iter += 1
loss_en = step(src_lg, batch_en, False)
loss_fr = step(tgt_lg, batch_fr,
n_iter % params.grad_acc_steps == 0)
if n_iter % 50 == 0:
time_taken = time.time() - start_time
hours, mins, secs = _to_hours_mins_secs(time_taken)
cur_lr = optimizer.get_lr()
logger.info(f" Iter {n_iter:>7} - MLM-{src_lg} {loss_en:.4f} -"
f" MLM-{tgt_lg} {loss_fr:.4f} - lr {cur_lr:.7f}"
f" elapsed {int(hours)}:{int(mins)}")
if n_iter % params.epoch_size == 0:
n_epoch += 1
valid_src_nll = evaluate(src_lg, valid_loader_en)
valid_tgt_nll = evaluate(tgt_lg, valid_loader_fr)
logger.info(
f" Validation - Iter {n_iter} |"
f" MLM-{src_lg} {valid_src_nll:.4f} MLM-{tgt_lg} {valid_tgt_nll:.4f}"
)
avg_nll = (valid_src_nll + valid_tgt_nll) / 2
if avg_nll < best_valid_nll:
best_valid_nll = avg_nll
logger.info(f"| Best checkpoint at epoch: {n_epoch}")
src_model = f'{model_prefix}_{src_lg}_ep{n_epoch}'
tgt_model = f'{model_prefix}_{tgt_lg}_ep{n_epoch}'
src_path = os.path.join(params.exp_path, src_model)
tgt_path = os.path.join(params.exp_path, tgt_model)
if not os.path.exists(src_path): os.makedirs(src_path)
if not os.path.exists(tgt_path): os.makedirs(tgt_path)
# save both models
logger.info(f'save ({src_lg}) model to: {src_path}')
model.src_model.save_pretrained(src_path)
logger.info(f'save ({tgt_lg}) model to: {tgt_path}')
model.tgt_model.save_pretrained(tgt_path)
if n_epoch == params.max_epoch:
exit()
np.seterr(all="ignore")
mpl.use('Agg')
mpl.rcParams['pdf.fonttype'] = 42
if any(
["Arial" in f.name for f in matplotlib.font_manager.fontManager.ttflist]):
mpl.rcParams['font.family'] = 'Arial'
__dir__ = os.path.dirname(os.path.abspath(__file__))
VERSION = "1.6.0"
LABEL = "pySashimi"
CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
init_logger("INFO")
def draw_without_bam(bam, count_table, splice_region, colors, color_factor,
threshold, output, sashimi_plot_settings, no_gene, dpi,
distance_ratio, title):
color_index = 0
tmp_color = {}
required_cols = {}
if bam:
with open(bam) as r:
for line in r:
lines = re.split(r"\t| {2,}", line.strip())
if not lines:
continue
def create_logger(self):
self.log_folder = os.path.join(os.getcwd(), 'logs',
self.utils.get_formatted_datetime())
init_logger(self.log_folder)