def __init__(self, username):
self.driver = config_driver()
self.username = str(username)
self.name = 'douban_' + self.username
config_log(self.name)
load_cookies(self.driver, self.name)
self.posts = douban['posts'].get(self.username)
self.up_posts()
def __init__(self, username):
self.driver = config_driver()
self.post_count = hupu['posts_count']
self.user_id = hupu['user_id']
self.commentaries = hupu['commentaries']
self.comment_count = 0
self.posts = Queue()
self.max_error_num = 5
self.username = str(username)
self.name = 'hupu_' + self.username
config_log(self.name)
load_cookies(self.driver, self.name)
self.comment_posts()
formatter_class=argparse.RawDescriptionHelpFormatter,
description=textwrap.dedent(
"""\
Fashion Hash Net Training Script
--------------------------------
Actions:
1. train: train fashion net.
2. evaluate: evaluate NDCG and accuracy.
3. retrieval: retrieval for items.
"""
),
)
actions = ACTION_FUNS.keys()
parser.add_argument("action", help="|".join(sorted(actions)))
parser.add_argument("--cfg", help="configuration file.")
args = parser.parse_args()
with open(args.cfg, "r") as f:
kwargs = yaml.load(f, Loader=yaml.FullLoader)
config = polyvore.param.FashionParam(**kwargs)
# config.add_timestamp()
logfile = utils.config_log(stream_level=config.log_level, log_file=config.log_file)
LOGGER.info("Logging to file %s", logfile)
LOGGER.info("Fashion param : %s", config)
if args.action in actions:
ACTION_FUNS[args.action](config)
exit(0)
else:
LOGGER.info("Action %s is not in %s", args.action, "|".join(actions))
exit(1)
# sequences = test_set.get_all_sequences()
# logger.debug("Sequences {}".format(sequences))
# logger.debug("Test words {}".format(test_set.wordlist))
# logger.debug("Sentences {}".format(test_set._load_sentence_word_indices()))
sentences = test_set._load_sentence_word_indices()
sentences_indexes = []
for k, v in sentences.items():
sentences_indexes.append(v)
sentences_indexes
# for test_word_index, test_word in enumerate(test_set.wordlist):
# for test_word_index in range(test_set.num_items):
for sentence_indexes in sentences_indexes:
sentence_score = get_sentence_score(sentence_indexes, models, test_set, probabilities, guesses, alpha_start, alpha_transition)
logger.debug("Sentence indexes {}".format(sentence_indexes))
logger.debug("Sentence score {}".format(sentence_score))
return probabilities, guesses
if __name__ == "__main__":
from asl_test_recognizer import TestRecognize
from utils import config_log
config_log()
logger = logging.getLogger('recognizer')
test_model = TestRecognize()
test_model.setUp()
test_model.test_recognize_probabilities_interface()
test_model.test_recognize_guesses_interface()
set_per_sub = 3 # number of image set per subject within a batch
pooling_type = 'NAN'
if feat_type == 'casianet':
feat_dim = 320
else:
feat_dim = 512
max_iter = 51000
test_iter = 50
save_dir = './data/IJBA/model/model_{}_s{}_{}'.format(feat_type, set_size,
pooling_type)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
logger = utils.config_log(save_dir,
'train_s{}b{}'.format(set_size, batch_size))
meta_dir = './data/IJBA/{}'.format(feat_type)
logger_result = open(os.path.join(save_dir, 'eval_result.txt'), 'w')
NAN_split_tars = np.zeros((10, 4), dtype=np.float32)
AVE_split_tars = np.zeros((10, 4), dtype=np.float32)
for idx_split in range(1, 11):
# lst_sub_faces = pickle.load(open(os.path.join(meta_dir, 'train_subject_{}.bin'.format(idx_split)), 'rb'))
label_qry = pickle.load(open('label_qry20200114145205rpc.p', 'rb'))
label_ref = pickle.load(open('label_ref20200115092149rpc.p', 'rb'))
feature_qry = torch.load('feature_qry20200114145205rpc.p')
feature_ref = torch.load('feature_ref20200115092149rpc.p')
lst_sub_faces = torch.cat((feature_qry, feature_ref), 0)
label = label_qry + label_ref
set_size = len(label_ref) // len(set(label_ref))
count_qry = len(label_qry)
num_epoch = parsed.num_epoch
save_dir = parsed.save_dir
train_filelist = parsed.train_filelist
root_dir = parsed.img_root_dir
pre_trained_model = parsed.pre_trained_model
gpus = parsed.gpus
train_batch_size = parsed.train_batch_size
step = parsed.step
input_image_size = 224
num_classes = 10575 - 25
lst_gpu = list(gpus.split(','))
os.environ['CUDA_VISIBLE_DEVICES'] = gpus
if not os.path.exists(save_dir):
os.makedirs(save_dir)
logger = utils.config_log(log_dir=save_dir,
fname='webface_resnet34_coso_train')
logger.info(parsed)
# image loader
train_transforms = read_utils.webface_train_transforms
train_img_loader = ImageFilelist(root_dir=root_dir,
fname=train_filelist,
transform=train_transforms)
train_loader = torch.utils.data.DataLoader(train_img_loader,
batch_size=train_batch_size,
shuffle=True,
num_workers=12)
# network
conv_net, spd_net = resnet.resnet34_cov(num_classes=num_classes,
norm_type='T',
def create_lmdb(dataset, image_dir):
LOGGER.info("Creating LMDB to %s", dataset)
image_list = utils.check.list_files(image_dir)
env = lmdb.open(dataset, map_size=2**40)
# open json file
with env.begin(write=True) as txn:
for image_name in tqdm(image_list):
fn = os.path.join(image_dir, image_name)
with open(fn, "rb") as f:
img_data = f.read()
txn.put(image_name.encode("ascii"), img_data)
env.close()
LOGGER.info("Converted Polyvore to LDMB")
if __name__ == "__main__":
LOGGER.info("Log to file %s", utils.config_log())
parser = argparse.ArgumentParser(description="Create LMDB")
parser.add_argument(
"src",
default="data/polyvore/images/291x291",
type=str,
help="image folder for polyvore dataset",
)
parser.add_argument("dst", type=str, help="folder to save lmdb")
args = parser.parse_args()
create_lmdb(args.dst, args.src)
exit(0)
num_epoch = parsed.num_epoch
save_dir = parsed.save_dir
train_filelist = parsed.train_filelist
root_dir = parsed.img_root_dir
pre_trained_model = parsed.pre_trained_model
gpus = parsed.gpus
train_batch_size = parsed.train_batch_size
step = parsed.step
input_image_size = 224
num_classes = 10575 - 25
os.environ['CUDA_VISIBLE_DEVICES'] = gpus
if not os.path.exists(save_dir):
os.makedirs(save_dir)
logger = utils.config_log(log_dir=save_dir, fname='train')
logger.info(parsed)
train_transforms = read_utils.webface_train_transforms
train_img_loader = ImageFilelist(root_dir=root_dir,
fname=train_filelist,
transform=train_transforms)
train_loader = torch.utils.data.DataLoader(train_img_loader,
batch_size=train_batch_size,
shuffle=True,
num_workers=8)
conv_net, spd_net = resnet.resnet34_cov(num_classes=num_classes,
norm_type='T',
num_iter=5)
r_loss = layer_utils.RingLoss(loss_weight=ring_weight)
