def single_res_training( epochs, model, optimizer, init_lr, lr_decay, train_loader, test_loader, loss_function, ): logger = getlogger() writer = SummaryWriter() logger.info("start training") for epoch in range(epochs): model.train() decay_lr(optimizer, epoch, lr_decay, 0.001) pbar = config_pbar(f"[EPOCH {epoch+1}]", train_loader) for i, (x, y) in pbar: x_val = x.cuda().float() y_val = y.cuda() - 1 if i == 0: x_image = vutils.make_grid(x_val[:1], scale_each=True) writer.add_image("train", x_image, i) optimizer.zero_grad() output = model(x_val) loss = loss_function(output, y_val) pbar.postfix[0]["loss"] = loss.item() loss.backward() optimizer.step() writer.add_scalar("training_loss", loss.item(), epoch + 1) if ((epoch + 1) % 10 != 0): continue model.eval() with torch.no_grad(): hit = 0 for i, (x, y) in tqdm(enumerate(test_loader)): x_val = torch.squeeze(x) x_val = x_val.cuda().float() y_val = y.cuda() - 1 if i == 0: x_image = vutils.make_grid(x_val[:1], scale_each=True) writer.add_image("val", x_image, i) output = model(x_val) output = nn.Softmax(dim=1)(output) output = torch.mean(output, dim=0) output = output.cpu().detach().numpy() if np.argmax(output) == y_val: hit = hit + 1 logger.info("hit : {}/{}, rate: {}%\n".format( hit, len(test_loader), float(hit / len(test_loader) * 100)))
def _build_cub_dataset( imagepath: str, labelpath: str, low_ratio: int, mean: list, std: list, batch_size: int = 128, shuffle=True, num_workers=6, drop_last=True, input_shape=(256, 256), crop_size=(224, 224), is_kd=False, is_test=False, ): logger = getlogger() params = { "batch_size": batch_size, "shuffle": shuffle, "num_workers": num_workers, "drop_last": drop_last } logger.info("SPECIFIC PREPROCESS") preprocess = build_transforms( input_shape=input_shape, crop_size=crop_size, mean=mean, std=std, low_ratio=low_ratio, is_test=is_test, ) if is_kd is True: logger.info("BASIC TRANSFORM") basic_transform = build_transforms( input_shape=input_shape, crop_size=crop_size, mean=mean, std=std, ) else: basic_transform = None dataset = CUB200_2011( imagepath=imagepath, labelpath=labelpath, preprocess=preprocess, basic_transform=basic_transform, is_kd=is_kd, is_test=is_test, ) return data.DataLoader(dataset, **params)
def Test(model, dataloaders): logger = getlogger() logger.info("Initiating Test...") _test_convnet(model) logger.info("...Model test complete") logger.info("\nValidating loaders...") # for dataloader in dataloaders: # _test_dataloader(dataloader) logger.info("...Done")
def _distillation_transform( input_shape: tuple, crop_size: tuple, mean, std, flip_prob=0.5, low_ratio=None, is_test=False, ): logger = getlogger() transform_list = [] transform_list.append(transforms.Resize(input_shape, interpolation=Image.BICUBIC)) if low_ratio is not None: transform_list.append(transforms.Resize(low_ratio, interpolation=Image.BICUBIC)) transform_list.append(transforms.Resize(input_shape, interpolation=Image.BICUBIC)) if is_test is True: transform_list.append(transforms.TenCrop(crop_size)) transform_list.append(transforms.Lambda(lambda crops: torch.stack([transforms.ToTensor()(crop) for crop in crops]))) transform_list.append(transforms.Lambda(lambda images: torch.stack([transforms.Normalize(mean=mean, std=std)(image) for image in images]))) for t in transform_list: logger.info("Transform : {}".format(t)) logger.info("\n") return transforms.Compose(transform_list) else: transform_list.append(transforms.RandomHorizontalFlip(flip_prob)) transform_list.append(transforms.RandomCrop(crop_size)) transform_list.append(transforms.ToTensor()) transform_list.append(transforms.Normalize(mean=mean, std=std)) for t in transform_list: logger.info("Transform : {}".format(t)) logger.info("\n") transform = transforms.Compose(transform_list) return transform
#3rd library import pysam, HTSeq import numpy as np import pandas as pd from joblib import Parallel, delayed import pylab import brewer2mpl #my own from utils import getlogger, call_sys #global settings #logger date = time.strftime(' %Y-%m-%d', time.localtime(time.time())) logger = getlogger(fn=os.getcwd() + "/" + date.strip() + "_" + os.path.basename(__file__) + ".log") #data REPEAT_GTF = "/picb/molsysbio/usr/caoyaqiang/1.Projects/17.Alu/1.GenomeReference/1.hg38/6.Repeats_Annotation/hg38_sorted_repeatmasker.gtf" CHROM_SIZE = "/home/caoyaqiang/caoyaqiang_han/1.Projects/17.Alu/1.GenomeReference/1.hg38/1.hg38_Sequence/hg38.chrom.sizes" RMSK = "/picb/molsysbio/usr/caoyaqiang/1.Projects/17.Alu/1.GenomeReference/1.hg38/6.Repeats_Annotation/rmsk.txt" SUBFAMILY_SIZE = "/picb/molsysbio/usr/caoyaqiang/1.Projects/17.Alu/1.GenomeReference/1.hg38/6.Repeats_Annotation/subfam.size" REPEATS_LOCUS = "/picb/molsysbio/usr/caoyaqiang/1.Projects/17.Alu/1.GenomeReference/1.hg38/6.Repeats_Annotation/1.GenomicLocation/hg38_rep_locus.txt" def iteres_stat_filter(bam, stat_root="0.iteres/1.stat/", filter_root="0.iteres/2.filter/"): sample = os.path.splitext(os.path.split(bam)[1])[0] if not os.path.exists(os.path.join(stat_root, sample)): os.makedirs(os.path.join(stat_root, sample)) stat_pre = os.path.join(stat_root, sample, sample)
import zerorpc from agent.config import MASTER_URL from .storage import Storage from utils import getlogger from .config import LOG_PATH from .storage import * import uuid logger = getlogger(__name__, LOG_PATH) class Agentserver: def __init__(self): self.store = Storage() def handle(self, msg): logger.info(type(msg)) try: if msg['type'] in {'register', 'heartbeat'}: print(1) payload = msg['payload'] print(2) info = {'hostname': payload['hostname'], 'ip': payload['ip']} print(3) self.store.reg_hb(payload['id'], info) print(4) logger.info('{}'.format(self.store.agents)) return 'ack {}'.format(msg) elif msg['type'] == 'result':
def main(args): logger = getlogger() logger.info("\nGENERATING MODEL") model = build_model( model_type=args.model_type, experiment_type=args.experiment_type, lr=args.lr, lr_decay=args.lr_decay, num_classes=args.num_classes, batch_size=args.batch_size, epochs=args.epochs, pretrain_path=args.pretrain_path, desc=args.desc, save=args.save, ) logger.info(model) logger.info("GENERATING TRAINLOADER") train_loader = build_dataloader( imagepath=args.imagepath, dataset_type=args.dataset_type, labelpath=args.train_label, low_ratio=args.low_ratio, batch_size=args.batch_size, mean=args.mean, std=args.std, is_kd=args.kd, is_test=False, ) logger.info("GENERATING TESTLOADER") test_loader = build_dataloader( imagepath=args.imagepath, dataset_type=args.dataset_type, labelpath=args.test_label, low_ratio=args.low_ratio, batch_size=1, mean=args.mean, std=args.std, is_kd=args.kd, is_test=True, ) default_optimizer = optim.SGD( model.parameters(), lr=args.lr, momentum=0.9, weight_decay=0.0005, ) logger.info(default_optimizer) Test(model, [train_loader, test_loader]) Baseline( TrainingConfig( model=model, epochs=args.epochs, optimizer=default_optimizer, init_lr=args.lr, train_loader=train_loader, valid_loader=test_loader, criterion=nn.CrossEntropyLoss(), ))
from subprocess import Popen, PIPE from utils import getlogger logger = getlogger(__name__, '/Users/quyixiao/PycharmProjects/mschedule/logs/exec.log') class Executor: def run(self, script, timeout=None): proc = Popen(script, shell=True, stdout=PIPE) code = proc.wait(timeout) txt = proc.stdout.read() logger.info("{} {}".format(code, txt)) return code,txt
from subprocess import PIPE, Popen from utils import getlogger logger = getlogger(__name__, 'd:/executor.log') class Executor: def run(self, script, timeout=None): process = Popen(script, shell=True, stdout=PIPE) code = process.wait() #状态码 output = process.stdout.read() #脚本执行输出 return code, output
import zerorpc import threading from .msg import Message from utils import getlogger from .config import LOG_PATH from .state import * from .executor import Executor logger = getlogger('__name__', LOG_PATH) class Agentclient: def __init__(self, url, message: Message): self.message = message self.client = zerorpc.Client() self.event = threading.Event() self.url = url self.state = WATTING #标记任务是否完成 self.executor = Executor() def start(self, itime=3): try: # 记录异常日志 self.event.clear() self.client.connect(self.url) reg = self.client.message(self.message.reg) print('reg: ', reg) logger.info(reg) while not self.event.wait(itime): server_message = self.client.message(self.message.heartbeat)
from .storage import Storage from utils import getlogger # agent[msg['payload']['id']] = msg['payload']['hostname'], msg.get('payload').get('ip') logger = getlogger(__name__, f'/tmp/{__name__}.log') class ConnectionManager: def __init__(self): self.store = Storage() def sendmsg(self, msg): # RPC 对外的接口,其中处理客户端传来的数据 try: if msg['type'] in {'register', 'heartbeat'}: self.store.reg_hb(**msg['payload']) elif msg['type'] == 'result': self.store.result(msg['payload']) logger.info(msg) return f"ACK: {msg}" except Exception as e: logger.error(e) return 'Bas Request' def add_task(self, msg: dict): return self.store.add_task(msg) def get_task(self, agent_id): return self.store.get_task(agent_id) def get_agents(self):
from utils import getlogger from .storage import Storage from .state import * import uuid logger = getlogger(__name__, 'd:/mastercm.log') class ConnectionManager: def __init__(self): self.store = Storage() def handle(self, msg): try: if msg['type'] in ('reg', 'heartbeat'): payload = msg['payload'] info = {'hostname': payload['hostname'], 'ip': payload['ip']} self.store.reg_hb(payload['id'], info) return "ack {}".format(msg) elif msg['type'] == 'result': payload = msg['payload'] agent_id = payload['agent_id'] task_id = payload['id'] state = SUCCEED if payload['output'] == 0 else FAILED output = payload['output'] task = self.store.get_task_by_agentid(task_id) t = task.targets[agent_id] t.state = state t.output = output
import zerorpc from .msg import Message import threading from utils import getlogger from .state import * from .executor import Executor logger = getlogger(__name__, 'd:/agentcm.log') class ConnectionManager: def __init__(self, master_url, message: Message): self.master_url = master_url self.message = message #这是一个对象,才能调实例方法 self.client = zerorpc.Client() self.event = threading.Event() self.state = WAITING #Agent的工作状态 self.exe = Executor() #脚本执行 def start(self, timeout=5): try: self.event.clear() #重置event self.client.connect(self.master_url) #连接 self._send(self.message.reg()) #发送注册消息 while not self.event.wait(timeout): self._send(self.message.heartbeat()) if self.state == WAITING: #Agent等待状态主动去Master拉任务 self._get_task(self.message.id) except Exception as e: