def _main():
last = ''
file_flag = False
while True:
file_flag = False
raw_data = input()
if raw_data == 'restart':
msw_queue.put(0)
break
if raw_data == 'exit':
msw_queue.put(1)
break
if raw_data == 'update':
raw_data = 'update:compress;update_to:*'
if raw_data == '1':
raw_data = 'ffmpeg:autostart'
if raw_data == '2':
raw_data = 'ffmpeg:enable;to:*,server:miku'
if raw_data == 'r':
raw_data = last
last = raw_data
if raw_data[:6] == '(file)': # like "(file)log: filename.exe"
raw_data = raw_data[6:]
file_flag = True
first_index, last_index = find_index(raw_data)
app = raw_data[:first_index]
body = raw_data[last_index:]
ihead = {}
if ';' in body and ':' in body:
ihead_index = body.index(';')
ihead_str = body[ihead_index + 1:]
body = body[:ihead_index]
ihead_list = ihead_str.split(',')
for key_value in ihead_list:
key, value = key_value.split(':')
ihead[key] = value
app = app.replace(' ', '')
dp = Datapack(head={'from': __name__})
dp.head.update(ihead)
dp.app = app
if file_flag:
dp.method = 'file'
dp.body = b''
dp.head['filename'] = body
else:
dp.body = body.encode()
send_queue.put(dp)
print('Command has been sent', dp)
def __init__(self):
if ONLYPROXY and not MYPROXY:
print('config failed because you set onlyproxy true but proxy false')
return
self.send_queue = queue.Queue()
self.id_dict = {}
self.lock = threading.Lock()
self.all_connection_list = []
self.wheel_queue = queue.Queue()
self.netlist = [] # store nagetive connection
self.netlist_pass = []
self.conflist = [] # store config connection
self.conflist_pass = []
self.mhtlist = [] # store exchanged connection
self.mhtlist_pass = []
self.proxydict = {}
self.alllist = [self.netlist, self.netlist_pass, self.conflist, self.conflist_pass, \
self.mhtlist, self.mhtlist_pass]
self.start_wheel_thread = threading.Thread(target=self.start_wheel, args=(), daemon=True)
self.start_wheel_thread.start()
self.start_accpet_connection_thread = threading.Thread(target=self.start_accpet_connection, args=(), daemon=True)
self.start_accpet_connection_thread.start()
self.start_sending_dp_thread = threading.Thread(target=self.start_sending_dp, args=(), daemon=True)
self.start_sending_dp_thread.start()
self.start_positive_connecting_thread = threading.Thread(target=self.start_positive_connecting, args=(), daemon=True)
self.start_positive_connecting_thread.start()
self.start_mht_thread = threading.Thread(target=self.start_mht, args=(), daemon=True)
self.start_mht_thread.start()
def send_to_id(self, to, dp): # send to 1 id, process proxy at the same time
connections = self.id_dict.get(to)
if not connections:
if to == ID:
print('To id %s is yourself!' % to, dp) # maybe proxy to yourself
return
if to in self.proxydict: # neat warning dangerous code
if not ID == self.proxydict[to]: # check whether proxy is yourself
if dp.head.get('to'):
dp.head['to'] = self.proxydict[to] + '&' + to + '&' + dp.head['to']
else:
dp.head['to'] = self.proxydict[to] + '&' + to
else:
if dp.head.get('to'):
dp.head['to'] = to + '&' + dp.head['to']
else:
dp.head['to'] = to
self.wheel_queue.put(dp)
return
print('To id %s has no connection now %d...' % (to, dp.failed_times), dp)
if dp.head.get('to'):
dp.head['to'] = to + '&' + dp.head['to']
else:
dp.head['to'] = to
self.wheel_queue.put(dp)
return
connection = connections[0]
connection.sendall(dp)
def start_wheel(self):
while True:
dp = self.wheel_queue.get()
dp.failed_times += 1
if dp.failed_times > 39:
print('Datapack abandom', dp)
continue
time.sleep(RETRYSLEEP)
receive_queue.put(dp)
def getlist(self, conntype):
if conntype == 'net':
return self.netlist, self.netlist_pass
elif conntype == 'conf':
return self.conflist, self.conflist_pass
elif conntype == 'mht':
return self.mhtlist, self.mhtlist_pass
else:
print('Could not find conntype %s' % conntype)
return None, None
def check_id(self):
'''
check id package must like
-------------------------------
post handshake msw/0.1
id: [yourID]
listen_port: [3900]
length: 0
-------------------------------
error code list:
1: not get "id" in head
2: receive data failed
3: appname is not handshake
4: id is yourself
'''
data = None
if self.positive:
self.send_id()
try:
data = self.conn.recv(BUFFSIZE)
except ConnectionResetError:
print('One connection failed before ID check')
if not data:
return 2, ''
self.buff += data
dp = Datapack()
dp.encode_data = self.buff # maybe here needs to use copy.copy(self.buff)
self.buff = dp.decode(only_head=True)
if not dp.head.get('id'):
return 1, dp.head.get('flag')
if not dp.app == 'handshake':
return 3, dp.head.get('flag')
self.id = dp.head['id']
self.listen_port = int(dp.head.get('listen_port'))
if self.id == ID:
#print('you connect to your self')
return 4, dp.head.get('flag')
if ONLYPROXY and not self.id == MYPROXY: # refuce not proxy connection
return 5, dp.head.get('flag')
if dp.head.get('onlyuseproxy'):
if not dp.head['onlyuseproxy'] == ID:
return 6, dp.head.get('flag')
if not self.positive:
self.send_id()
return 0, dp.head.get('flag')
def process_command(self, dp):
if dp.body == b'status':
result = ''
result += 'Online %s' % str(self.id_dict) + '\n'
result += 'proxydict %s' % str(self.proxydict) + '\n'
result += 'conflist %s' % str(self.conflist) + '\n'
result += 'conflist_pass %s' % str(self.conflist_pass) + '\n'
result += 'netlist %s' % str(self.netlist) + '\n'
result += 'netlist_pass %s' % str(self.netlist_pass) + '\n'
result += 'mhtlist %s' % str(self.mhtlist) + '\n'
result += 'mhtlist_pass %s' % str(self.mhtlist_pass)
ndp = dp.reply()
ndp.body = result.encode()
send_queue.put(ndp)
elif dp.body == b'mht' and dp.method == 'get':
ndp = dp.reply()
data_dict = {}
connection_list = []
with self.lock:
for id in self.id_dict:
connections = self.id_dict[id]
for connection in connections:
ip, port = connection.conn.getpeername()
port = int(connection.listen_port)
connection_list.append((ip, port))
for addr in self.conflist:
if not addr in connection_list:
connection_list.append(addr)
for addr in self.conflist_pass:
if not addr in connection_list:
connection_list.append(addr)
data_dict['mht'] = connection_list
data_dict['proxy'] = self.proxydict
ndp.body = json.dumps(data_dict).encode()
send_queue.put(ndp)
elif dp.method == 'reply':
mhtstr = dp.body.decode()
data_dict = json.loads(mhtstr)
mhtlist = data_dict['mht']
with self.lock:
for addr in mhtlist:
addr = (addr[0], addr[1])
if not self.check_in_list(addr):
self.mhtlist.append(addr)
self.proxydict.update(data_dict['proxy'])
else:
print('Received unknown command', dp)
def main():
while True:
dp = receive_queue.get()
if dp.method == 'file':
word = dp.head.get('filename')
else:
word = dp.body.decode()
print('Writedown log: %s' % (word), dp)
with open('logger.log', 'a') as f:
if dp.head.get('from'):
from_app_name = dp.head.get('from')
else:
from_app_name = 'Unknown'
f.write(from_app_name + ': ' + dp.body.decode() + '\n')
def set_connection(self, connection):
id = connection.id
with self.lock:
if not self.id_dict.get(id):
self.id_dict[id] = []
self.id_dict[id].append(connection)
self.all_connection_list.append(connection)
xxxlist, xxxlist_pass = self.getlist(connection.conntype)
addr = (connection.addr[0], connection.listen_port)
if addr in xxxlist:
xxxlist.remove(addr)
if not addr in xxxlist_pass:
xxxlist_pass.append(addr)
print('<%s> %s connected' % (connection.flag, id))
def del_connection(self, connection):
id = connection.id
with self.lock:
self.id_dict[id].remove(connection)
self.all_connection_list.remove(connection)
if id in self.id_dict and not self.id_dict.get(id): # del the empty user
del(self.id_dict[id])
if connection.listen_port: # avoid "None" addr port
xxxlist, xxxlist_pass = self.getlist(connection.conntype)
addr = (connection.addr[0], connection.listen_port)
if not addr in xxxlist:
xxxlist.append(addr)
if addr in xxxlist_pass:
xxxlist_pass.remove(addr)
print('<%s> %s disconnected' % (connection.flag, id))
def start_accpet_connection(self):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
listen_ip = jsondata.try_to_read_jsondata('listen_ip', '127.0.0.1')
listen_port = jsondata.try_to_read_jsondata('listen_port', 3900)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind((listen_ip, listen_port))
listen_num = jsondata.try_to_read_jsondata('listen_num', 39)
s.listen(listen_num)
print('Sucessfully listen at %s:%s, max connection:%s' % (listen_ip, listen_port, listen_num))
while True:
conn, addr = s.accept()
connection = Connection(conn, addr, self)
connection.i_did_something()
def read_addrlist(self):
if not os.path.exists('addrlist.txt'):
print('addrlist.txt not exists, config that base on addrlist_sample.txt')
else:
with open('addrlist.txt', 'r') as f:
raw_data = f.read()
raw_data = raw_data.replace('\r', '')
lines = raw_data.split('\n')
while '' in lines:
lines.remove('')
for line in lines:
ip, port = line.split(':')
ip = socket.gethostbyname(ip)
port = int(port)
self.conflist.append((ip, port))
if jsondata.try_to_read_jsondata('proxy', False):
self.proxydict[ID] = jsondata.raw_jsondata['proxy']
def send_func(self):
while True:
dp = self.padding_queue.get()
dp.encode()
self.conn.sendall(dp.encode_data)
if dp.method == 'file':
with open(dp.head['filename'], 'rb') as f:
for data in f:
try:
self.conn.sendall(data)
except Exception as e:
print('Failed to send file %s %s: %s' % (dp.head['filename'], type(e), str(e)), dp)
if dp.head.get('to'):
dp.head['to'] = self.id + '&' + dp.head['to']
else:
dp.head['to'] = self.id
self.netowrk_controller.wheel_queue.put(dp)
break
if dp.delete:
os.remove(dp.head['filename'])
print('Send file %s to %s finished' % (dp.head['filename'], self.id), dp)
def main(model_path, backbone, scale, path, save_path, gpu_id):
device = torch.device("cuda:" + str(gpu_id))
logger = setup_logger(os.path.join(config.output_dir, 'test_log'))
logger.info(config.print())
if os.path.exists(save_path):
shutil.rmtree(save_path, ignore_errors=True)
if not os.path.exists(save_path):
os.makedirs(save_path)
save_img_folder = os.path.join(save_path, 'img')
if not os.path.exists(save_img_folder):
os.makedirs(save_img_folder)
save_txt_folder = os.path.join(save_path, 'result')
if not os.path.exists(save_txt_folder):
os.makedirs(save_txt_folder)
img_paths = [os.path.join(path, x) for x in os.listdir(path)]
net = PSENet(backbone=backbone,
pretrained=config.pretrained,
result_num=config.n)
model = Pytorch_model(model_path, net=net, scale=scale, gpu_id=gpu_id)
num_gpus = torch.cuda.device_count()
if num_gpus > 1:
model = nn.DataParallel(model)
model = model.to(device)
model = nn.DataParallel(model)
recall, precision, f1 = merge_eval(model=model,
save_path=os.path.join(
config.output_dir, 'output'),
test_path=config.testroot,
device=device,
base_path=config.base_path,
use_sub=config.use_sub)
logger.info('test: recall: {:.6f}, precision: {:.6f}, f1: {:.6f}'.format(
recall, precision, f1))
# total_frame = 0.0
# total_time = 0.0
# for img_path in tqdm(img_paths):
# img_name = os.path.basename(img_path).split('.')[0]
# save_name = os.path.join(save_txt_folder, 'res_' + img_name + '.txt')
# _, boxes_list, t = model.predict(img_path)
# total_frame += 1
# total_time += t
# # img = draw_bbox(img_path, boxes_list, color=(0, 0, 255))
# # cv2.imwrite(os.path.join(save_img_folder, '{}.jpg'.format(img_name)), img)
# np.savetxt(save_name, boxes_list.reshape(-1, 8), delimiter=',', fmt='%d')
# print('fps:{}'.format(total_frame / total_time))
return save_txt_folder
def __init__(self):
help(config._config)
arguments = self.parse_arguments()
if arguments.print_config:
print(config.print(config.default_config()))
return
if not arguments.config_file:
print("Must specify configuration file via -c. If no configuration"
+ " file exists, you can generate a blank one with the -p"
+ " flag")
return
try:
self.config = config.parse(arguments.config_file)
except config.BadConfiguration:
print("Your configuration file is invalid. To generate a new,"
+ " blank configuration, use the -p flag.")
def start_sending_dp(self):
while True:
dp = receive_queue.get()
if dp.app == 'net' and not dp.head.get('to'):
self.process_command(dp)
continue
if not dp.head.get('to'):
print('You got a no head datapack')
print(str(dp.head))
continue
to_str = dp.head['to']
to_list = to_str.split('&')
to = to_list.pop(0)
to_str = '&'.join(to_list)
dp.head['to'] = to_str
if to == '*':
with self.lock:
for id in self.id_dict:
connection = self.id_dict[id][0]
connection.sendall(dp)
elif not to:
print('not to', dp)
elif ONLYPROXY and not to == MYPROXY:
if dp.head['to']:
dp.head['to'] = to + dp.head['to']
else:
dp.head['to'] = to
self.send_to_id(MYPROXY, dp)
else:
self.send_to_id(to, dp)
def receive(self):
still_need = 0
while True:
try:
data = self.conn.recv(BUFFSIZE)
except ConnectionResetError:
break
except Exception as e:
print('Connection recv error %s: %s' % (type(e), str(e)))
break
if not data:
break
self.buff += data
if not still_need:
dp = Datapack()
dp.encode_data = self.buff
try:
self.buff = dp.decode(only_head=True)
if dp.method == 'file':
create_floder(dp.head['filename'])
create_floder('tmp/' + dp.head['filename'])
self.f = open('tmp/' + dp.head['filename'], 'ab')
if dp.method == 'file' and os.path.exists(dp.head['filename']):
os.remove(dp.head['filename'])
except Exception as e:
print('Decode head failed %s: %s' % (type(e), str(e)))
print(self.buff)
break
length = int(dp.head.get('length'))
still_need = length
if still_need > len(self.buff):
# writing tmp data
if dp.method == 'file':
still_need -= self.f.write(self.buff)
else:
dp.body += self.buff
still_need -= len(self.buff)
self.buff = b'' # empty buff because all tmp data has been write
else: # download complete setuation
if dp.method == 'file':
self.f.write(self.buff[:still_need])
self.f.close()
self.f = None
else:
dp.body = self.buff[:still_need]
self.buff = self.buff[still_need:]
still_need = 0
# bleow code are using to process datapack
if dp.method == 'file':
os.rename('tmp/' + dp.head['filename'], dp.head['filename'])
print('Received file %s from %s' % (dp.head['filename'], self.id), dp)
send_queue.put(dp)
# below code are using to closed connection
if self.f:
self.f.close()
self.f = None
self.conn.close()
self.netowrk_controller.del_connection(self)
def main():
while True:
try:
_main()
except Exception as e:
print('Error in %s, %s: %s' % (__name__, type(e), str(e)))
def print_reply_func():
while True:
dp = receive_queue.get()
dp.encode()
print(dp.encode_data.decode())
import adb
import config
import run_one
# Read config
config = config.ReadConfig("./python/config.json")
config.print()
# Push model files into adb
adb_runner = adb.AdbRunner("./python/output.txt")
adb_runner.push_files("./resource/<model name>")
# Run One
one_runner = run_one.OneRunner(config)
#adb_runner.run("""ls -all
#echo "hi"
#echo "nice to meet you"
#""", clean = True)
#adb_runner.close()
# run
def main():
if config.output_dir is None:
config.output_dir = 'output'
if config.restart_training:
shutil.rmtree(config.output_dir, ignore_errors=True)
if not os.path.exists(config.output_dir):
os.makedirs(config.output_dir)
logger = setup_logger(os.path.join(config.output_dir, 'train_log'))
logger.info(config.print())
torch.manual_seed(config.seed) # 为CPU设置随机种子
if config.gpu_id is not None and torch.cuda.is_available():
torch.backends.cudnn.benchmark = True
logger.info('train with gpu {} and pytorch {}'.format(config.gpu_id, torch.__version__))
device = torch.device("cuda:0")
torch.cuda.manual_seed(config.seed) # 为当前GPU设置随机种子
torch.cuda.manual_seed_all(config.seed) # 为所有GPU设置随机种子
else:
logger.info('train with cpu and pytorch {}'.format(torch.__version__))
device = torch.device("cpu")
train_data = MyDataset(config.trainroot, config.MIN_LEN, config.MAX_LEN, transform=transforms.ToTensor())
train_loader = Data.DataLoader(dataset=train_data, batch_size=config.train_batch_size, shuffle=True,
num_workers=int(config.workers))
writer = SummaryWriter(config.output_dir)
model = CTPN_Model(pretrained=config.pretrained)
if not config.pretrained and not config.restart_training:
model.apply(weights_init)
num_gpus = torch.cuda.device_count()
if num_gpus > 1:
model = nn.DataParallel(model)
model = model.to(device)
dummy_input = torch.zeros(1, 3, 600, 800).to(device)
writer.add_graph(model=model, input_to_model=dummy_input)
criterion = CTPNLoss(device)
# optimizer = torch.optim.SGD(model.parameters(), lr=config.lr, momentum=0.99)
optimizer = torch.optim.Adam(model.parameters(), lr=config.lr)
if config.checkpoint != '' and not config.restart_training:
print('Loading Checkpoint...')
start_epoch = load_checkpoint(config.ch9eckpoint, model, logger, device)
start_epoch += 1
scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, config.lr_decay_step, gamma=config.lr_gamma,
last_epoch=start_epoch)
else:
start_epoch = config.start_epoch
scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, config.lr_decay_step, gamma=config.lr_gamma)
all_step = len(train_loader)
logger.info('train dataset has {} samples,{} in dataloader'.format(train_data.__len__(), all_step))
epoch = 0
best_model = {'loss': float('inf')}
try:
for epoch in range(start_epoch, config.epochs):
start = time.time()
train_loss, lr = train_epoch(model, optimizer, scheduler, train_loader, device, criterion, epoch, all_step,
writer, logger)
logger.info('[{}/{}], train_loss: {:.4f}, time: {:.4f}, lr: {}'.format(
epoch, config.epochs, train_loss, time.time() - start, lr))
# if (0.3 < train_loss < 0.4 and epoch % 1 == 0) or train_loss < 0.3:
if epoch % 10 == 0 or train_loss < best_model['loss']:
net_save_path = '{}/PSENet_{}_loss{:.6f}.pth'.format(config.output_dir, epoch, train_loss)
save_checkpoint(net_save_path, model, optimizer, epoch, logger)
if train_loss < best_model['loss']:
best_model['loss'] = train_loss
if 'model' in best_model:
os.remove(best_model['model'])
best_model['model'] = net_save_path
shutil.copy(best_model['model'],
'{}/best_loss{:.6f}.pth'.format(config.output_dir, best_model['loss']))
writer.close()
except KeyboardInterrupt:
pass
finally:
if best_model['model']:
shutil.copy(best_model['model'], '{}/best_loss{:.6f}.pth'.format(config.output_dir, best_model['loss']))
logger.info(best_model)
def run_as_server(self):
_padding_to_convert = os.listdir('res/ffmpeg_tmp')
padding_to_convert = []
for file in _padding_to_convert:
file = 'res/ffmpeg_tmp/' + file
padding_to_convert.append(file)
already_in_convert = []
finished_convert = [] # outputfilename
while True:
dp = receive_queue.get()
if dp.method == 'post' and dp.body == b'status':
result = ''
result += 'padding_to_convert ' + str(
padding_to_convert) + '\n'
result += 'already_in_convert ' + str(
already_in_convert) + '\n'
result += 'finished_convert ' + str(finished_convert) + '\n'
result += 'convert_task_queue size ' + str(
self.convert_task_queue.qsize())
ndp = dp.reply()
ndp.body = result.encode()
send_queue.put(ndp)
elif dp.method == 'post' and dp.body == b'reset':
padding_to_convert = already_in_convert
already_in_convert = []
elif dp.method == 'post' and dp.body == b'stop':
break
elif dp.method == 'post' and dp.body == b'pause':
self.pause = True
elif dp.method == 'post' and dp.body == b'continue':
self.pause = False
elif dp.method == 'get':
if self.pause:
ndp = dp.reply()
ndp.method = 'post'
ndp.body = b'disable'
send_queue.put(ndp)
continue
if padding_to_convert:
filename = padding_to_convert.pop(0)
already_in_convert.append(filename)
print('%s get %s to convert' % (dp.head['id'], filename),
dp)
ndp = dp.reply()
ndp.method = 'file'
ndp.head['filename'] = filename
send_queue.put(ndp)
else:
if not already_in_convert: # finished
break
else: # waiting for final convert
ndp = dp.reply()
ndp.method = 'post'
ndp.body = b'disable'
send_queue.put(ndp)
elif dp.method == 'file':
old_filename = dp.head['old_filename']
filename = dp.head['filename']
os.remove(old_filename)
already_in_convert.remove(old_filename)
finished_convert.append(filename)
total = len(padding_to_convert) + len(
already_in_convert) + len(finished_convert)
print('Processing...(%d) %d/%d %s' % \
(len(already_in_convert), \
len(finished_convert), \
total, \
str(round(len(finished_convert)/total*100, 2))))
if not padding_to_convert and not already_in_convert: # final process
break
print('Mapreduce finished')
def mainloop(self):
_create_floder('res/ffmpeg_tmp')
_create_floder('res/ffmpeg_finished')
_create_floder('res/ffmpeg_task')
_create_floder('res/ffmpeg_old')
_create_floder('res/ffmpeg_complet')
while True:
dp = receive_queue.get()
if dp.method == 'post' and dp.body == b'concat':
self.org_filename = dp.head['filename']
self.object_filename = self.org_filename[:-4] + '.mkv'
self.concat_func()
if dp.method == 'post' and dp.body == b'autostart':
filelist = os.listdir('res/ffmpeg_task')
self.tasklist = []
for file in filelist:
if len(file) > 3:
ext = file[-4:]
if ext in ['.mp4', '.MP4', '.mkv', '.MKV']:
self.tasklist.append('res/ffmpeg_task/' + file)
dp = Datapack()
dp.app = 'ffmpeg'
dp.body = b'start'
dp.head['filename'] = self.tasklist.pop(0)
self.autostart = dp.head['filename']
send_queue.put(dp)
if dp.method == 'post' and dp.body == b'start': # config ffmpeg is server or client
if dp.head.get('concat'):
if dp.head['concat'] == 'true':
self.concat = True
elif dp.head['concat'] == 'false':
self.concat = False
else:
print('unknown concat value')
continue
else:
self.concat = True
if self.concat:
self.org_filename = dp.head['filename']
self.object_filename = 'res/ffmpeg_complet/' + os.path.basename(
self.org_filename)[:-4] + '.mkv'
if self.concat:
ndp = dp.reply()
ndp.body = 'Spliting file %s' % dp.head['filename']
ndp.body = ndp.body.encode()
send_queue.put(ndp)
cmd = 'ffmpeg -i "' + os.path.normpath(
dp.head['filename']) + '" -c copy \
-f segment -segment_time 20 -reset_timestamps 1 -y \
"res/ffmpeg_tmp/' + '%d' + '.mkv"'
os.system(cmd)
self.run_as_server()
if self.concat:
self.concat_func()
print('All process finished')
elif dp.method == 'post' and dp.body == b'enable': # clinet mode
self.status = 1
self.server = dp.head['server']
self.convert_func()
elif dp.method == 'post' and dp.body == b'status':
result = 'ffmpeg not working'
ndp = dp.reply()
ndp.body = result.encode()
send_queue.put(ndp)
elif dp.method == 'get': # let other client disable
ndp = dp.reply()
ndp.method = 'post'
ndp.body = b'disable'
print('let %s disabled' % dp.head['id'])
send_queue.put(ndp)
def main():
if config.output_dir is None:
config.output_dir = 'output'
if config.restart_training:
shutil.rmtree(config.output_dir, ignore_errors=True)
if not os.path.exists(config.output_dir):
os.makedirs(config.output_dir)
logger = setup_logger(os.path.join(config.output_dir, 'train_log'))
logger.info(config.print())
torch.manual_seed(config.seed) # 为CPU设置随机种子
if config.gpu_id is not None and torch.cuda.is_available():
torch.backends.cudnn.benchmark = True
logger.info('train with gpu {} and pytorch {}'.format(
config.gpu_id, torch.__version__))
device = torch.device("cuda:0")
torch.cuda.manual_seed(config.seed) # 为当前GPU设置随机种子
torch.cuda.manual_seed_all(config.seed) # 为所有GPU设置随机种子
else:
logger.info('train with cpu and pytorch {}'.format(torch.__version__))
device = torch.device("cpu")
train_data = TibetanDataset(config.json_path,
data_shape=config.data_shape,
n=config.n,
m=config.m,
transform=transforms.ToTensor(),
base_path=config.base_path)
train_loader = Data.DataLoader(dataset=train_data,
batch_size=config.train_batch_size,
shuffle=True,
num_workers=int(config.workers))
writer = SummaryWriter(config.output_dir)
model = PSENet(backbone=config.backbone,
pretrained=config.pretrained,
result_num=config.n,
scale=config.scale)
if not config.pretrained and not config.restart_training:
model.apply(weights_init)
num_gpus = torch.cuda.device_count()
if num_gpus > 1:
model = nn.DataParallel(model)
model = model.to(device)
# dummy_input = torch.autograd.Variable(torch.Tensor(1, 3, 600, 800).to(device))
# writer.add_graph(models=models, input_to_model=dummy_input)
criterion = PSELoss(Lambda=config.Lambda,
ratio=config.OHEM_ratio,
reduction='mean')
# optimizer = torch.optim.SGD(models.parameters(), lr=config.lr, momentum=0.99)
optimizer = torch.optim.Adam(model.parameters(), lr=config.lr)
if config.checkpoint != '' and not config.restart_training:
start_epoch = load_checkpoint(config.checkpoint, model, logger, device,
optimizer)
start_epoch += 1
scheduler = torch.optim.lr_scheduler.MultiStepLR(
optimizer,
config.lr_decay_step,
gamma=config.lr_gamma,
last_epoch=start_epoch)
else:
start_epoch = config.start_epoch
scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer,
config.lr_decay_step,
gamma=config.lr_gamma)
all_step = len(train_loader)
logger.info('train dataset has {} samples,{} in dataloader'.format(
train_data.__len__(), all_step))
epoch = 0
best_model = {'recall': 0, 'precision': 0, 'f1': 0, 'models': ''}
try:
for epoch in range(start_epoch, config.epochs):
start = time.time()
train_loss, lr = train_epoch(model, optimizer, scheduler,
train_loader, device, criterion,
epoch, all_step, writer, logger)
logger.info(
'[{}/{}], train_loss: {:.4f}, time: {:.4f}, lr: {}'.format(
epoch, config.epochs, train_loss,
time.time() - start, lr))
# net_save_path = '{}/PSENet_{}_loss{:.6f}.pth'.format(config.output_dir, epoch,
# train_loss)
# save_checkpoint(net_save_path, models, optimizer, epoch, logger)
if (0.3 < train_loss < 0.4 and epoch % 4 == 0) or train_loss < 0.3:
recall, precision, f1 = merge_eval(model=model,
save_path=os.path.join(
config.output_dir,
'output'),
test_path=config.testroot,
device=device,
base_path=config.base_path,
use_sub=config.use_sub)
logger.info(
'test: recall: {:.6f}, precision: {:.6f}, f1: {:.6f}'.
format(recall, precision, f1))
net_save_path = '{}/PSENet_{}_loss{:.6f}_r{:.6f}_p{:.6f}_f1{:.6f}.pth'.format(
config.output_dir, epoch, train_loss, recall, precision,
f1)
save_checkpoint(net_save_path, model, optimizer, epoch, logger)
if f1 > best_model['f1']:
best_path = glob.glob(config.output_dir + '/Best_*.pth')
for b_path in best_path:
if os.path.exists(b_path):
os.remove(b_path)
best_model['recall'] = recall
best_model['precision'] = precision
best_model['f1'] = f1
best_model['models'] = net_save_path
best_save_path = '{}/Best_{}_r{:.6f}_p{:.6f}_f1{:.6f}.pth'.format(
config.output_dir, epoch, recall, precision, f1)
if os.path.exists(net_save_path):
shutil.copyfile(net_save_path, best_save_path)
else:
save_checkpoint(best_save_path, model, optimizer,
epoch, logger)
pse_path = glob.glob(config.output_dir + '/PSENet_*.pth')
for p_path in pse_path:
if os.path.exists(p_path):
os.remove(p_path)
writer.add_scalar(tag='Test/recall',
scalar_value=recall,
global_step=epoch)
writer.add_scalar(tag='Test/precision',
scalar_value=precision,
global_step=epoch)
writer.add_scalar(tag='Test/f1',
scalar_value=f1,
global_step=epoch)
writer.close()
except KeyboardInterrupt:
save_checkpoint('{}/final.pth'.format(config.output_dir), model,
optimizer, epoch, logger)
finally:
if best_model['models']:
logger.info(best_model)
def main():
if config.output_dir is None:
config.output_dir = 'output'
if config.restart_training:
shutil.rmtree(config.output_dir, ignore_errors=True)
if not os.path.exists(config.output_dir):
os.makedirs(config.output_dir)
logger = setup_logger(os.path.join(config.output_dir, 'train_log'))
logger.info(config.print())
torch.manual_seed(config.seed) # 为CPU设置随机种子
if config.gpu_id is not None and torch.cuda.is_available():
torch.backends.cudnn.benchmark = True
logger.info('train with gpu {} and pytorch {}'.format(
config.gpu_id, torch.__version__))
device = torch.device("cuda:0")
torch.cuda.manual_seed(config.seed) # 为当前GPU设置随机种子
torch.cuda.manual_seed_all(config.seed) # 为所有GPU设置随机种子
else:
logger.info('train with cpu and pytorch {}'.format(torch.__version__))
device = torch.device("cpu")
train_data = MyDataset(config.trainroot,
data_shape=config.data_shape,
n=config.n,
m=config.m,
transform=transforms.ToTensor())
train_loader = Data.DataLoader(dataset=train_data,
batch_size=config.train_batch_size,
shuffle=True,
num_workers=int(config.workers),
drop_last=True)
writer = SummaryWriter(config.output_dir)
model = PSENet(backbone=config.backbone,
pretrained=config.pretrained,
result_num=config.n,
scale=config.scale)
if not config.pretrained and not config.restart_training:
model.apply(weights_init)
## loading the pretrained weights from drive
state_dict = torch.load(config.pretrained_path)
model.load_state_dict(state_dict)
num_gpus = torch.cuda.device_count()
if num_gpus > 1:
model = nn.DataParallel(model)
model = model.to(device)
# dummy_input = torch.autograd.Variable(torch.Tensor(1, 3, 600, 800).to(device))
# writer.add_graph(models=models, input_to_model=dummy_input)
criterion = PSELoss(Lambda=config.Lambda,
ratio=config.OHEM_ratio,
reduction='mean')
# optimizer = torch.optim.SGD(models.parameters(), lr=config.lr, momentum=0.99)
optimizer = torch.optim.Adam(model.parameters(), lr=config.lr)
if config.checkpoint != '' and not config.restart_training:
start_epoch = load_checkpoint(config.checkpoint, model, logger, device)
start_epoch += 1
scheduler = torch.optim.lr_scheduler.MultiStepLR(
optimizer,
config.lr_decay_step,
gamma=config.lr_gamma,
last_epoch=start_epoch)
else:
start_epoch = config.start_epoch
scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer,
config.lr_decay_step,
gamma=config.lr_gamma)
all_step = len(train_loader)
logger.info('train dataset has {} samples,{} in dataloader'.format(
train_data.__len__(), all_step))
epoch = 0
try:
for epoch in range(start_epoch, config.epochs):
start = time.time()
train_loss, lr = train_epoch(model, optimizer, scheduler,
train_loader, device, criterion,
epoch, all_step, writer, logger)
logger.info(
'[{}/{}], train_loss: {:.4f}, time: {:.4f}, lr: {}'.format(
epoch, config.epochs, train_loss,
time.time() - start, lr))
# net_save_path = '{}/PSENet_{}_loss{:.6f}.pth'.format(config.output_dir, epoch,
# train_loss)
# save_checkpoint(net_save_path, models, optimizer, epoch, logger)
state_dict = model.state_dict()
# replace the weight file
filename = '{}/PSENet_resnet50.pth'.format(config.output_dir)
if os.path.exists(filename):
os.unlink(filename)
torch.save(state_dict, filename)
writer.close()
except KeyboardInterrupt:
filename = '{}/PSENet_resnet50.pth'.format(config.output_dir)
if os.path.exists(filename):
os.unlink(filename)
torch.save(state_dict, filename)
save_checkpoint('{}/final.pth'.format(config.output_dir), model,
optimizer, epoch, logger)
