def testCalculateChgNum(self):
with open(
os.path.join(UtilPath.Test(), "testDataset",
self.repository["name"], "chgNum.csv")) as f:
reader = csv.reader(f)
datasTest = [row for row in reader][1:]
for dataTest in datasTest:
pathFile = dataTest[18].replace("/", "\\")
with self.subTest(pathFile=pathFile):
data = Data(self.gr, pathFile,
self.dataset.getCommitsBug(self.repository))
self.assertEqual(data.calculateChgNum(), int(dataTest[3]))
@Author: Zengrui Zhao
"""
import torch
import sys
import time
from torch.utils.data import DataLoader
sys.path.append('..')
from src.dataset import Data
if __name__ == '__main__':
use_cuda = torch.cuda.is_available()
for num_workers in range(1, 16): # 遍历worker数
kwargs = {
'num_workers': num_workers,
'pin_memory': False
} if use_cuda else {}
train_set = Data(rootPth='/home/zzr/Data/multiNucle', mode='test')
train_loader = DataLoader(train_set,
batch_size=16,
drop_last=False,
shuffle=True,
**kwargs)
start = time.time()
for epoch in range(1, 3):
for batch_idx, (data, target) in enumerate(train_loader): # 不断load
pass
end = time.time()
print("Finish with:{} second, num_workers={}".format(
end - start, num_workers))
datetime.date.today() + datetime.timedelta(days=1)
).strftime("%Y/%m/%d")
try:
book_page = client.request_booking_page()
img_resp = client.request_security_code_img(book_page.content)
image_ori = np.array(Image.open(io.BytesIO(img_resp.content)))
image = cv2.resize(image_ori, (128, 128))
with torch.no_grad():
image = (
cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)[np.newaxis, np.newaxis, :, :] / 255.0
)
image = torch.tensor(image, dtype=torch.float32).cuda()
code = CNN.decode(model(image))[0]
pred = Data.decode(code)
params["homeCaptcha:securityCode"] = "".join(pred)
### show debug image
# cv2.putText(image_ori, "".join(pred), (20, 20), 3, 1, (255, 0, 255))
# cv2.imshow("img", image_ori)
# cv2.waitKey(0)
result = client.submit_booking_form(params)
errors = error_feedback.parse(result.content)
if len(errors) == 0:
cv2.imwrite(
os.path.join(target_path, "ori", str(image_cnt).zfill(5) + ".jpg"), image_ori
)
crawled_data.append([str(image_cnt).zfill(5) + ".jpg", "".join(pred)])
def main(args, logger):
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
writer = SummaryWriter(logdir=args.subTensorboardDir)
trainSet = Data(mode='train')
trainLoader = DataLoader(trainSet,
batch_size=args.batchSizeTrain,
shuffle=True,
pin_memory=False,
drop_last=False,
num_workers=args.numWorkers)
testSet = Data(mode='test')
testLoader = DataLoader(testSet,
batch_size=args.batchSizeTest,
shuffle=False,
pin_memory=False,
num_workers=args.numWorkers)
# net = smp.Unet(classes=2).to(device)
net = smp.Linknet(classes=1,
activation='sigmoid',
encoder_name='se_resnext101_32x4d').to(device)
# criterion = nn.CrossEntropyLoss().to(device)
criterion = smploss.DiceLoss(eps=sys.float_info.min).to(device)
# criterion = DiceLoss().to(device)
optimizer = optim.SGD(net.parameters(), lr=.1, momentum=.9)
scheduler = optim.lr_scheduler.StepLR(optimizer,
step_size=args.scheduleStep,
gamma=0.1)
runningLoss = []
st = stGloble = time.time()
totalIter = len(trainLoader) * args.epoch
iter = 0
for epoch in range(args.epoch):
if epoch != 0 and epoch % args.evalFrequency == 0:
pass
if epoch != 0 and epoch % args.saveFrequency == 0:
modelName = osp.join(args.subModelDir, 'out_{}.pth'.format(epoch))
state_dict = net.modules.state_dict() if hasattr(
net, 'module') else net.state_dict()
torch.save(state_dict, modelName)
for img, mask in trainLoader:
iter += 1
img = img.to(device)
mask = mask.to(device, dtype=torch.int64).unsqueeze(1)
optimizer.zero_grad()
outputs = net(img)
# print(outputs.shape, mask.shape)
# break
loss = criterion(outputs, mask)
loss.backward()
optimizer.step()
runningLoss.append(loss.item())
if iter % args.msgFrequency == 0:
# writer.add_images('img', img, iter)
# writer.add_images('mask', mask.unsqueeze(1), iter)
ed = time.time()
spend = ed - st
spendGloable = ed - stGloble
st = ed
eta = int((totalIter - iter) * (spendGloable / iter))
spendGloable = str(datetime.timedelta(seconds=spendGloable))
eta = str(datetime.timedelta(seconds=eta))
avgLoss = np.mean(runningLoss)
runningLoss = []
lr = optimizer.param_groups[0]['lr']
msg = '. '.join([
'epoch:{epoch}', 'iter/total_iter:{iter}/{totalIter}',
'lr:{lr:.5f}', 'loss:{loss:.4f}',
'spend/gloable_spend:{spend:.4f}/{gloable_spend}',
'eta:{eta}'
]).format(epoch=epoch,
loss=avgLoss,
iter=iter,
totalIter=totalIter,
spend=spend,
gloable_spend=spendGloable,
lr=lr,
eta=eta)
logger.info(msg)
writer.add_scalar('loss', avgLoss, iter)
writer.add_scalar('lr', lr, iter)
scheduler.step()
outName = osp.join(args.subModelDir, 'final.pth')
torch.save(net.cpu().state_dict(), outName)