def check_rank(self, av_id: int, times=0):
rank_list = self.rank_map[av_id] if av_id in self.rank_map else []
changeHeaders({'Referer': self.BASIC_AV_URL % av_id})
if len(rank_list):
score = int(rank_list[1])
rank = int(rank_list[0])
url = self.ARCHIVE_STAT_URL % av_id
json_req = get_request_proxy(url, 1)
if not self.have_error(json_req):
if times < 3:
self.check_rank(av_id, times + 1)
return
json_req = json_req['data']
need = [
'view', 'like', 'coin', 'favorite', 'reply', 'share', 'danmaku'
]
data = [json_req[index] for index in need]
if not self.check_view(av_id, data[0]):
if times < 3:
self.check_rank(av_id, times + 1)
return
if len(rank_list):
data = [time_str(), *data, *rank_list[:2], *rank_list[3:5]]
else:
data = [time_str(), *data]
with codecs.open('%s%d.csv' % (history_dir, av_id),
'a',
encoding='utf-8') as f:
f.write(','.join([str(index) for index in data]) + '\n')
if self.check_rank_list(av_id, rank_list):
av_id_id = int(av_id) * 10 + int(rank_list[-1])
if av_id_id not in self.rank:
self.rank[av_id_id] = [rank_list[0] // 10]
else:
self.rank[av_id_id].append(rank_list[0] // 10)
self.last_rank[av_id_id] = rank_list[0]
send_email(
'%dday List || Rank: %d Score: %d' %
(int(rank_list[-1]), rank, score),
'%dday List || Rank: %d Score: %d' %
(int(rank_list[-1]), rank, score))
if av_id in self.last_check and self.last_check[av_id] - int(
time.time()) > one_day:
del self.rank_map[av_id]
elif av_id not in self.last_check and int(
time.time()) > one_day + self.begin_timestamp:
del self.rank_map[av_id]
self.last_view[av_id] = data[1]
def get_star_num(self, mid: int, times: int, load_disk=False):
''' get star num'''
url = self.RELATION_STAT_URL % mid
header = {
**headers,
**{
'Origin': self.BILIBILI_URL,
'Referer': self.AV_URL
}
}
if 'Host' in header:
del header['Host']
req = get_request_proxy(url, 2, header=header)
if req is None or req.status_code != 200 or len(
req.text) < 8 or not '{' in req.text:
if times < 3:
self.get_star_num(mid, times + 1, load_disk)
return
try:
json_req = json.loads(req.text[7:-1])
self.star[mid] = json_req['data']['follower']
if load_disk and self.check_star(mid, self.star[mid]):
self.last_star[mid] = self.star[mid]
with open('{}star.csv'.format(data_dir), 'a') as f:
f.write('%s,%d\n' % (time_str(), self.star[mid]))
except:
pass
def public_monitor(self, av_id: int, times: int):
''' a monitor '''
self.public_list.append(av_id)
data_time, mid = self.public[av_id]
self.get_star_num(mid, 0)
self.check_rank_v2(av_id, 0)
time.sleep(5)
follower = self.star[mid] if mid in self.star else 0
origin_data = self.data_v2[av_id] if av_id in self.data_v2 else []
sleep_time = data_time + one_day - int(time.time())
if sleep_time < 0:
return
print('Monitor Begin %d' % (av_id))
time.sleep(sleep_time)
self.get_star_num(mid, 0)
self.check_rank_v2(av_id, 0)
time.sleep(5)
follower_2 = self.star[mid] if mid in self.star else 0
one_day_data = self.data_v2[av_id] if av_id in self.data_v2 else []
data = [
time_str(data_time), av_id, follower, follower_2, *origin_data,
*one_day_data
]
with codecs.open(data_dir + 'public.csv', 'a', encoding='utf-8') as f:
f.write(','.join([str(ii) for ii in data]) + '\n')
def generate_other_params(self,
hotel_id: int = 4889292,
city_id: int = 2,
startDate: str = time_str(-1, '%Y-%m-%d'),
depDate: str = time_str(
int(time.time() + one_day), '%Y-%m-%d')):
''' generate other params '''
params = {
'psid': None,
'MasterHotelID': hotel_id,
'hotel': hotel_id,
'EDM': 'F',
'roomId': None,
'IncludeRoom': None,
'city': city_id,
'showspothotel': 'T',
'supplier': None,
'IsDecoupleSpotHotelAndGroup': 'F',
'contrast': 0,
'brand': 776,
'startDate': startDate,
'depDate': depDate,
'IsFlash': 'F',
'RequestTravelMoney': 'F',
'hsids': None,
'IsJustConfirm': None,
'contyped': 0,
'priceInfo': -1,
'equip': None,
'filter': None,
'productcode': None,
'couponList': None,
'abForHuaZhu': None,
'defaultLoad': 'T',
'esfiltertag': None,
'estagid': None,
'Currency': None,
'Exchange': None,
'minRoomId': 0,
'maskDiscount': 0,
'TmFromList': 'F',
'th': 119,
'RoomGuestCount': '1,1,0'
}
return params
def get_check(self):
''' check comment '''
now_hour = int(time_str(format='%H'))
now_min = int(time_str(format='%M'))
now_time = now_hour + now_min / 60
if now_time > 0.5 and now_time < 8.5:
return
if os.path.exists('{}comment.pkl'.format(comment_dir)):
with codecs.open('{}comment.pkl'.format(comment_dir), 'rb') as f:
self.comment = pickle.load(f)
if self.assign_up_mid == -1:
return
url = self.MEMBER_SUBMIT_URL % self.assign_up_mid
json_req = get_request_proxy(url, 1)
if json_req is None or not 'data' in json_req or not 'vlist' in json_req[
'data']:
if can_retry(url):
self.get_check()
return
av_id_list = [[ii['aid'], ii['comment']]
for ii in json_req['data']['vlist']]
if self.basic_av_id not in [ii[0] for ii in av_id_list]:
if can_retry(url):
self.get_check()
return
threading_list = []
for (ii, jj) in av_id_list:
if ii not in self.comment:
self.comment[ii] = {}
work = threading.Thread(target=self.comment_check_schedule,
args=(
ii,
jj,
))
threading_list.append(work)
for work in threading_list:
work.start()
for work in threading_list:
work.join()
with codecs.open('{}comment.pkl'.format(comment_dir), 'wb') as f:
pickle.dump(self.comment, f)
return av_id_list
def check_rank_v2(self, av_id: int, times=0):
rank_list = self.rank_map[av_id] if av_id in self.rank_map else []
changeHeaders({'Referer': self.BASIC_AV_URL % av_id})
url = self.ARCHIVE_STAT_URL % av_id
json_req = get_request_proxy(url, 1)
if not self.have_error(json_req):
if times < 3:
self.check_rank_v2(av_id, times + 1)
return
json_req = json_req['data']
need = [
'view', 'like', 'coin', 'favorite', 'reply', 'share', 'danmaku'
]
data = [json_req[index] for index in need]
if len(rank_list):
data = [time_str(), *data, *rank_list[:2], *rank_list[-2:]]
else:
data = [time_str(), *data]
self.data_v2[av_id] = data
def get_comment_detail(self,
comment: dict,
av_id: int,
pn: int,
parent_floor=None):
''' get comment detail '''
ctime = time_str(comment['ctime'])
wait_list = ['floor', 'member', 'content', 'like']
wait_list_mem = ['uname', 'sex', 'sign', 'level_info']
wait_list_content = ['message', 'plat']
floor, member, content, like = [comment[ii] for ii in wait_list]
uname, sex, sign, level = [member[ii] for ii in wait_list_mem]
current_level = level['current_level']
content, plat = [content[ii] for ii in wait_list_content]
req_list = [
floor, ctime, like, plat, current_level, uname, sex, content, sign
]
self.have_bad_comment(req_list, av_id, pn, parent_floor)
req_list[-1] = req_list[-1].replace(',', ' ').replace('\n', ' ')
req_list[-2] = req_list[-2].replace(',', ' ').replace('\n', ' ')
return req_list
def _setup_default_logging(self, default_level=logging.INFO):
format = "%(asctime)s - %(levelname)s - %(name)s - %(message)s"
dataset_name = get_dataset_name(self.config.dataset_path)
output_dir = os.path.join(dataset_name, f'x{self.L}')
os.makedirs(output_dir, exist_ok=True)
writer = SummaryWriter(comment=f'{dataset_name}_{self.L}')
logger = logging.getLogger('train')
logger.setLevel(default_level)
time_stamp = time_str()
logging.basicConfig( # unlike the root logger, a custom logger can’t be configured using basicConfig()
filename=os.path.join(output_dir, f'{time_stamp}_{self.L}_labelled_instances.log'),
format=format,
datefmt="%m/%d/%Y %H:%M:%S",
level=default_level)
# to avoid double printing when creating new instances of class
if not logger.handlers:
console_handler = logging.StreamHandler(sys.stdout)
console_handler.setLevel(default_level)
console_handler.setFormatter(logging.Formatter(format))
logger.addHandler(console_handler)
#
logger.info(dict(self.config._get_kwargs()))
if self.device != 'cpu':
logger.info(f'Device used: {self.device}_{torch.cuda.get_device_name(self.device)}')
logger.info(f'Model: {self.model.module.__class__ if isinstance(self.model, torch.nn.DataParallel) else self.model.__class__}')
logger.info(f'Num_labels: {self.L}')
logger.info(f'Image_size: {self.config.im_size}')
logger.info(f'Cropsize: {self.config.cropsize}')
logger.info("Total params: {:.2f}M".format(
sum(p.numel() for p in self.model.parameters()) / 1e6))
return logger, writer, time_stamp
def log_write(self, url):
"""
failure log
"""
with codecs.open("proxy.log", 'a', encoding='utf-8') as f:
f.write(time_str() + url + '\n')
def __init__(self):
parser = argparse.ArgumentParser()
parser.add_argument('-d', '--sys_device_ids', type=eval, default=(0, ))
parser.add_argument('-r', '--run', type=int, default=1)
parser.add_argument('--set_seed', type=str2bool, default=False)
parser.add_argument(
'--dataset',
type=str,
default='market1501',
choices=['market1501', 'cuhk03', 'duke', 'combined'])
parser.add_argument('--trainset_part',
type=str,
default='trainval',
choices=['trainval', 'train'])
parser.add_argument('--model_type',
type=str,
default='densenet121',
choices=[
'resnet50', 'densenet121', 'preActResnet50',
'resnet50mid'
])
parser.add_argument('--apply_random_erasing',
type=str2bool,
default=False)
parser.add_argument('--resize_h_w', type=eval, default=(256, 128))
# These several only for training set
parser.add_argument('--crop_prob', type=float, default=0)
parser.add_argument('--crop_ratio', type=float, default=1)
parser.add_argument('--mirror', type=str2bool, default=True)
parser.add_argument('--ids_per_batch', type=int, default=32)
parser.add_argument('--ims_per_id', type=int, default=4)
parser.add_argument('--log_to_file', type=str2bool, default=True)
parser.add_argument('--steps_per_log', type=int, default=20)
parser.add_argument('--epochs_per_val', type=int, default=1e10)
parser.add_argument('--last_conv_stride',
type=int,
default=1,
choices=[1, 2])
parser.add_argument('--normalize_feature',
type=str2bool,
default=False)
parser.add_argument('--margin', type=float, default=0.3)
parser.add_argument('--only_test', type=str2bool, default=False)
parser.add_argument('--resume', type=str2bool, default=False)
parser.add_argument('--exp_dir', type=str, default='')
parser.add_argument('--model_weight_file', type=str, default='')
parser.add_argument('--base_lr', type=float, default=2e-4)
parser.add_argument('--lr_decay_type',
type=str,
default='exp',
choices=['exp', 'staircase'])
parser.add_argument('--exp_decay_at_epoch', type=int, default=151)
parser.add_argument('--staircase_decay_at_epochs',
type=eval,
default=(
101,
201,
))
parser.add_argument('--staircase_decay_multiply_factor',
type=float,
default=0.1)
parser.add_argument('--total_epochs', type=int, default=300)
parser.add_argument(
'--softmax_loss_weight',
default=0.9,
type=float,
help='weight assign to softmax loss between 0 and 1')
parser.add_argument(
'--add_softmax_loss',
default=True,
type=bool,
help='loss will be combination of triplet and softmax loss')
args = parser.parse_args()
# gpu ids
self.sys_device_ids = args.sys_device_ids
# If you want to make your results exactly reproducible, you have
# to fix a random seed.
if args.set_seed:
self.seed = 1
else:
self.seed = None
# The experiments can be run for several times and performances be averaged.
# `run` starts from `1`, not `0`.
self.run = args.run
###########
# Dataset #
###########
# If you want to make your results exactly reproducible, you have
# to also set num of threads to 1 during training.
if self.seed is not None:
self.prefetch_threads = 1
else:
self.prefetch_threads = 2
self.dataset = args.dataset
self.trainset_part = args.trainset_part
# Image Processing
# Just for training set
self.crop_prob = args.crop_prob
self.crop_ratio = args.crop_ratio
self.resize_h_w = args.resize_h_w
# Whether to scale by 1/255
self.scale_im = True
self.im_mean = [0.486, 0.459, 0.408]
self.im_std = [0.229, 0.224, 0.225]
self.train_mirror_type = 'random' if args.mirror else None
self.ids_per_batch = args.ids_per_batch
self.ims_per_id = args.ims_per_id
self.train_final_batch = False
self.train_shuffle = True
self.random_erasing = args.apply_random_erasing
self.test_batch_size = 32
self.test_final_batch = True
self.test_mirror_type = None
self.test_shuffle = False
dataset_kwargs = dict(name=self.dataset,
resize_h_w=self.resize_h_w,
scale=self.scale_im,
im_mean=self.im_mean,
im_std=self.im_std,
batch_dims='NCHW',
num_prefetch_threads=self.prefetch_threads)
prng = np.random
if self.seed is not None:
prng = np.random.RandomState(self.seed)
self.train_set_kwargs = dict(part=self.trainset_part,
ids_per_batch=self.ids_per_batch,
ims_per_id=self.ims_per_id,
final_batch=self.train_final_batch,
shuffle=self.train_shuffle,
crop_prob=self.crop_prob,
crop_ratio=self.crop_ratio,
mirror_type=self.train_mirror_type,
is_random_erasing=self.random_erasing,
prng=prng)
self.train_set_kwargs.update(dataset_kwargs)
prng = np.random
if self.seed is not None:
prng = np.random.RandomState(self.seed)
self.val_set_kwargs = dict(part='val',
batch_size=self.test_batch_size,
final_batch=self.test_final_batch,
shuffle=self.test_shuffle,
mirror_type=self.test_mirror_type,
prng=prng)
self.val_set_kwargs.update(dataset_kwargs)
###############
# ReID Model #
###############
# The last block of ResNet has stride 2. We can set the stride to 1 so that
# the spatial resolution before global pooling is doubled.
self.last_conv_stride = args.last_conv_stride
# Whether to normalize feature to unit length along the Channel dimension,
# before computing distance
self.normalize_feature = args.normalize_feature
# Margin of triplet loss
self.margin = args.margin
#############
# Training #
#############
self.weight_decay = 0.0005
# Initial learning rate
self.base_lr = args.base_lr
self.lr_decay_type = args.lr_decay_type
self.exp_decay_at_epoch = args.exp_decay_at_epoch
self.staircase_decay_at_epochs = args.staircase_decay_at_epochs
self.staircase_decay_multiply_factor = args.staircase_decay_multiply_factor
# Number of epochs to train
self.total_epochs = args.total_epochs
# How often (in epochs) to test on val set.
self.epochs_per_val = args.epochs_per_val
# How often (in batches) to log. If only need to log the average
# information for each epoch, set this to a large value, e.g. 1e10.
self.steps_per_log = args.steps_per_log
# Only test and without training.
self.only_test = args.only_test
self.resume = args.resume
#######
# Log #
#######
# If True,
# 1) stdout and stderr will be redirected to file,
# 2) training loss etc will be written to tensorboard,
# 3) checkpoint will be saved
self.log_to_file = args.log_to_file
# The root dir of logs.
if args.exp_dir == '':
self.exp_dir = osp.join(
'exp/train',
'{}'.format(self.dataset),
#
'lcs_{}_'.format(self.last_conv_stride) +
('nf_' if self.normalize_feature else 'not_nf_') +
'margin_{}_'.format(tfs(self.margin)) +
'lr_{}_'.format(tfs(self.base_lr)) +
'{}_'.format(self.lr_decay_type) +
('decay_at_{}_'.format(self.exp_decay_at_epoch) if self.
lr_decay_type == 'exp' else 'decay_at_{}_factor_{}_'.format(
'_'.join([str(e)
for e in args.staircase_decay_at_epochs]),
tfs(self.staircase_decay_multiply_factor))) +
'total_{}'.format(self.total_epochs),
#
'run{}'.format(self.run),
)
else:
self.exp_dir = args.exp_dir
self.stdout_file = osp.join(self.exp_dir,
'stdout_{}.txt'.format(time_str()))
self.stderr_file = osp.join(self.exp_dir,
'stderr_{}.txt'.format(time_str()))
# Saving model weights and optimizer states, for resuming.
self.ckpt_file = osp.join(self.exp_dir, 'ckpt.pth')
self.model_type = args.model_type
# Just for loading a pretrained model; no optimizer states is needed.
self.model_weight_file = args.model_weight_file
# usage of softmax
self.softmax_loss_weight = args.softmax_loss_weight
self.add_softmax_loss = args.add_softmax_loss
def __init__(self):
parser = argparse.ArgumentParser()
parser.add_argument('-d', '--sys_device_ids', type=eval, default=(0, ))
parser.add_argument('--dataset',
type=str,
default='market1501',
choices=['market1501', 'cuhk03', 'duke'])
parser.add_argument('--num_queries', type=int, default=16)
parser.add_argument('--rank_list_size', type=int, default=10)
parser.add_argument('--resize_h_w', type=eval, default=(256, 128))
parser.add_argument('--last_conv_stride',
type=int,
default=1,
choices=[1, 2])
parser.add_argument('--normalize_feature',
type=str2bool,
default=False)
parser.add_argument('--log_to_file', type=str2bool, default=True)
parser.add_argument('--exp_dir', type=str, default='')
parser.add_argument('--ckpt_file', type=str, default='')
parser.add_argument('--model_weight_file', type=str, default='')
parser.add_argument('--opt-level',
type=str,
default='O0',
choices=['O0', 'O1', 'O2', 'O3'])
parser.add_argument(
'--net',
type=str,
default='mobilenetV2',
choices=['resnet50', 'shuffelnetV2', 'mobilenetV2'])
args = parser.parse_args()
# gpu ids
self.sys_device_ids = args.sys_device_ids
self.num_queries = args.num_queries
self.rank_list_size = args.rank_list_size
###########
# Dataset #
###########
self.dataset = args.dataset
self.prefetch_threads = 2
# Image Processing
self.resize_h_w = args.resize_h_w
# Whether to scale by 1/255
self.scale_im = True
self.im_mean = [0.486, 0.459, 0.408]
self.im_std = [0.229, 0.224, 0.225]
self.test_mirror_type = None
self.test_batch_size = 32
self.test_final_batch = True
self.test_shuffle = False
dataset_kwargs = dict(name=self.dataset,
resize_h_w=self.resize_h_w,
scale=self.scale_im,
im_mean=self.im_mean,
im_std=self.im_std,
batch_dims='NCHW',
num_prefetch_threads=self.prefetch_threads)
prng = np.random
self.test_set_kwargs = dict(part='test',
batch_size=self.test_batch_size,
final_batch=self.test_final_batch,
shuffle=self.test_shuffle,
mirror_type=self.test_mirror_type,
prng=prng)
self.test_set_kwargs.update(dataset_kwargs)
###############
# ReID Model #
###############
# The last block of ResNet has stride 2. We can set the stride to 1 so that
# the spatial resolution before global pooling is doubled.
self.last_conv_stride = args.last_conv_stride
self.opt_level = args.opt_level
self.net = args.net
# Whether to normalize feature to unit length along the Channel dimension,
# before computing distance
self.normalize_feature = args.normalize_feature
#######
# Log #
#######
# If True, stdout and stderr will be redirected to file
self.log_to_file = args.log_to_file
# The root dir of logs.
if args.exp_dir == '':
self.exp_dir = osp.join(
'exp/visualize_rank_list',
'{}'.format(self.dataset),
)
else:
self.exp_dir = args.exp_dir
self.stdout_file = osp.join(self.exp_dir,
'stdout_{}.txt'.format(time_str()))
self.stderr_file = osp.join(self.exp_dir,
'stderr_{}.txt'.format(time_str()))
# Model weights and optimizer states, for resuming.
self.ckpt_file = args.ckpt_file
# Just for loading a pretrained model; no optimizer states is needed.
self.model_weight_file = args.model_weight_file
# update weights; a gradient descent step
optimizer.step()
############
# step log #
############
# log loss for this batch
loss_meter.update(to_scalar(loss))
# write to tensorboard; used for visualization
writer.add_scalar('train/total_loss_iter', to_scalar(loss),
step + 1 + dataset_L * epoch)
# print the log for every "steps_per_log" batches or the final batch
if (step + 1) % cfg.steps_per_log == 0 or (step +
1) % len(train_loader) == 0:
log = '{}, Step {}/{} in Ep {}, {:.2f}s, loss:{:.4f}'.format( \
time_str(), step+1, dataset_L, epoch+1, time.time()-step_st, loss_meter.val)
print(log)
# update the learning rate
scheduler.step()
##############
# epoch log #
##############
# add tensorboard log for this epoch
writer.add_scalar('train/total_avgloss_epoch', loss_meter.avg, epoch + 1)
# print the log for this epoch
log = 'Ep{}, {:.2f}s, loss {:.4f}'.format(epoch + 1,
time.time() - ep_st,
loss_meter.avg)
print(log)
def __init__(self):
parser = argparse.ArgumentParser()
# Seed for generating random numbers
parser.add_argument('--set_seed', type=str2bool, default=False)
# Specify the device, gpu/cpu; -1 for cpu ; >0 for gpu
parser.add_argument('-d', '--sys_device_ids', type=eval, default=())
## dataset parameter
# the data file that has not been spliited into train, val, test set
parser.add_argument('--raw_data', type=str, default='')
# the path the the folder that saves the data file
parser.add_argument('--data_root', type=str, default='')
# the dataset name
parser.add_argument('--dataset',
type=str,
default='Beyer1986v1',
choices=[
'Beyer1986v1', 'Beyer1986v2', 'Beyer1986v3',
'data_FF', 'values_FAB', 'data_tot'
])
# batchsize for training the data
parser.add_argument('--batch_size', type=int, default=4)
# Number of workers to fetch the data parallely
parser.add_argument('--workers', type=int, default=2)
# How many data will be used for training set
parser.add_argument('--portion_train', type=float, default=0.7)
# How many data will be used for validation set
parser.add_argument('--portion_val', type=float, default=0.15)
## model
# the dim of inputs
parser.add_argument('--num_input', type=int, default=3)
# the dim of output
parser.add_argument('--num_output', type=int, default=2)
# how many hidden layers does the model have
parser.add_argument('--num_h', type=int, default=1)
# wight decay for stochastic gradient descent algorithm
parser.add_argument('--sgd_weight_decay', type=float, default=0.0005)
# momentum for for stochastic gradient descent algorithm
parser.add_argument('--sgd_momentum', type=float, default=0.9)
# learning rate
parser.add_argument('--lr', type=float, default=0.001)
# the number of neurons per hidden layer
parser.add_argument('--neurons_per_hlayer',
type=int,
nargs='+',
default=[3, 8])
# decay the learning rate for every "--epochs_per_decay" epochs
parser.add_argument('--epochs_per_decay',
type=int,
nargs='+',
default=[10, 15])
parser.add_argument('--staircase_decay_multiple_factor',
type=float,
default=0.1)
# total number of epochs to train
parser.add_argument('--total_epochs', type=int, default=10)
## utils
# the name for the saved model
parser.add_argument('--ckpt_file', type=str, default='')
# whether load the model weight from an existing ckpt file
parser.add_argument('--load_model_weight',
type=str2bool,
default=False)
# only test the model without training; you need to provide model path
parser.add_argument('--test_only', type=str2bool, default=False)
# the folder to store the experiment logs, model weights, etc
parser.add_argument('--exp_dir', type=str, default='')
# whether save the generated logs during training to a file
parser.add_argument('--log_to_file', type=str2bool, default=True)
#
parser.add_argument('--steps_per_log', type=int, default=2)
# validation the current model for every '--epochs_per_val' epochs
parser.add_argument('--epochs_per_val', type=int, default=10)
# save the model to disk for every '--epochs_per_val' epochs
parser.add_argument('--epochs_per_save', type=int, default=50)
# the index for the expeiments. integer required
parser.add_argument('--run', type=int, default=1)
# parse the arguments
args = parser.parse_args()
# add those arguemnt as attributes of the class
self.sys_device_ids = args.sys_device_ids
# random
self.set_seed = args.set_seed
if self.set_seed:
self.rand_seed = 0
else:
self.rand_seed = None
# run time index
self.run = args.run
# Dataset #
self.data_root = args.data_root
self.dataset_name = args.dataset
self.raw_data = args.raw_data
self.mean = [0.485, 0.456, 0.406]
self.std = [0.229, 0.224, 0.225]
self.batch_size = args.batch_size
self.workers = args.workers
self.input_size = args.num_input
self.output_size = args.num_output
self.portion_train = args.portion_train
self.portion_val = args.portion_val
# optimization
self.sgd_momentum = args.sgd_momentum
self.sgd_weight_decay = args.sgd_weight_decay
self.lr = args.lr
self.epochs_per_decay = args.epochs_per_decay
self.staircase_decay_multiple_factor = args.staircase_decay_multiple_factor
self.total_epochs = args.total_epochs
# utils
self.ckpt_file = args.ckpt_file
self.load_model_weight = args.load_model_weight
if self.load_model_weight:
if self.ckpt_file == '':
print(
'Please input the ckpt_file if you want to resume training'
)
raise ValueError
self.test_only = args.test_only
self.exp_dir = args.exp_dir
self.log_to_file = args.log_to_file
self.steps_per_log = args.steps_per_log
self.epochs_per_val = args.epochs_per_val
self.epochs_per_save = args.epochs_per_save
self.run = args.run
# for model
model_kwargs = dict()
model_kwargs['input_size'] = self.input_size
model_kwargs['output_size'] = self.output_size
model_kwargs['num_h'] = args.num_h
model_kwargs['hidden_size'] = args.neurons_per_hlayer
self.model_kwargs = model_kwargs
# for evaluation
# create folder for experiments
if self.exp_dir == '':
self.exp_dir = os.path.join('exp', '{}'.format(self.dataset_name),
'run{}'.format(self.run))
# the txt file the store the logs printed during training
self.stdout_file = os.path.join(self.exp_dir, \
'log', 'stdout_{}.txt'.format(time_str()))
self.stderr_file = os.path.join(self.exp_dir, \
'log', 'stderr_{}.txt'.format(time_str()))
may_mkdir(self.stdout_file)
def get_data(self, origin_data: list) -> str:
''' get data '''
return ','.join(str(ii) for ii in [time_str(), *origin_data]) + '\n'
