def extract_refer_spoc(thread_num=1):
src_base = '/home1/hvr2/crct_new/train_refer_vgg_feature_conv5_3'
out_base = '/home1/hvr2/crct_spoc3/train_refer_spoc3'
dlist = []
dir_cnt = 0
if not os.path.exists(out_base): os.makedirs(out_base)
t0 = time.time()
fop.recursive_dir(src_base, dlist, depth=0)
print "Dir number: ", len(dlist)
task_list = []
total = len(dlist)
if total <= thread_num: thread_num = total
each = total / thread_num
for tid in range(thread_num):
start = tid * each + 1
end = start + each - 1
if tid + 1 == thread_num: end = total
task = pop.task(func=thread_train_spoc,
args=(tid + 1, (start, end), dlist, out_base))
task.run()
task_list.append((tid + 1, task))
for tid, task in task_list:
task.close()
print "Task [%d] exited." % tid
t1 = time.time()
print("src_base: %s" % src_base)
print("out_base: %s" % out_base)
print("Total time: %d sec" % (t1 - t0))
print("--- extract refer spoc feat End ---")
def save_crow_feat(file_list=None):
src_base = os.path.join(my_root, 'train_refer_feat_alex')
out_base = os.path.join(my_root, 'train_refer_crop_feat_crow')
dlist = []
dir_cnt = 0
t0 = time.time()
fop.recursive_dir(src_base, dlist, depth=0)
model = caffe_model(model_id=model_id, model_init=0)
for dir in dlist:
dir_cnt += 1
if file_list is not None:
if dir_cnt < file_list[0] or dir_cnt > file_list[1]: continue
flist = os.listdir(dir)
print(" --- Crow feat: %d: %s frame: %d" %
(dir_cnt, dir, len(flist)))
out_dir = os.path.join(out_base, dir.split('/')[-1])
if not os.path.exists(out_dir): os.makedirs(out_dir)
for f in flist:
src = os.path.join(dir, f)
x = np.load(src)
crow = model.crow_aggregation(x)
dst = os.path.join(out_dir, src.split('/')[-1])
np.save(dst, crow)
#print("===SHAPE:",x.shape,crow.shape,crow);exit()
t1 = time.time()
print("Total time: %d sec" % (t1 - t0))
def extract_spoc_feat(file_list=None, model_id=0, thread_num=1):
if model_id == 1: return
prefix = ['train_refer_', 'train_refer_crop_']
src_sufix = ['vgg', '', 'alex']
dst_sufix = ['vggspoc', '', 'alexspoc']
model = caffe_model(model_id=model_id, model_init=0)
print "--- extract spoc feat Start ---"
for cnt in range(2):
src_base = os.path.join(my_root, prefix[cnt] + src_sufix[model_id])
out_base = os.path.join(my_root, prefix[cnt] + dst_sufix[model_id])
dlist = []
dir_cnt = 0
if not os.path.exists(out_base): os.makedirs(out_base)
t0 = time.time()
fop.recursive_dir(src_base, dlist, depth=0)
print "Dir number: ", len(dlist)
task_list = []
total = len(dlist)
if total <= thread_num: thread_num = total
each = total / thread_num
for tid in range(thread_num):
start = tid * each + 1
end = start + each - 1
if tid + 1 == thread_num: end = total
task = pop.task(func=thread_spoc,
args=(tid + 1, model_id, (start, end), dlist,
out_base))
task.run()
task_list.append((tid + 1, task))
for tid, task in task_list:
task.close()
print "Task [%d] exited." % tid
t1 = time.time()
print("src_base: %s" % src_base)
print("out_base: %s" % out_base)
print("model_id: %d, sigma: %.1f" % (model_id, model.sigma))
print("Total time: %d sec" % (t1 - t0))
print("--- extract spoc feat End ---")
def extract_test_spoc(thread_num=1, test=1):
if test == 1: # test set
src_base = os.path.join(my_root, 'test2_query_vgg')
out_base = os.path.join(my_root, 'test2_query_spoc3')
else: # train set
src_base = '/home1/hvr2/copyright_competition/train_query_vgg_conv5_3'
out_base = os.path.join(my_root, 'train_query_spoc3')
dlist = []
dir_cnt = 0
if not os.path.exists(out_base): os.makedirs(out_base)
t0 = time.time()
fop.recursive_dir(src_base, dlist, depth=0)
print "Dir number: ", len(dlist)
task_list = []
total = len(dlist)
if total <= thread_num: thread_num = total
each = total / thread_num
for tid in range(thread_num):
start = tid * each + 1
end = start + each - 1
if tid + 1 == thread_num: end = total
task = pop.task(func=thread_test_spoc,
args=(tid + 1, (start, end), dlist, out_base))
task.run()
task_list.append((tid + 1, task))
for tid, task in task_list:
task.close()
print "Task [%d] exited." % tid
t1 = time.time()
print("src_base: %s" % src_base)
print("out_base: %s" % out_base)
print("Total time: %d sec" % (t1 - t0))
print("--- extract test spoc feat End ---")
def extract_conv_feat(file_list=None, model_id=0):
model = caffe_model(model_id=model_id)
if model_id == 0: suffix = 'vgg'
elif model_id == 2: suffix = 'alex'
for cnt in range(2):
if cnt:
src_base = os.path.join(my_root, 'train_refer_crop_image')
out_base = os.path.join(my_root, 'train_refer_crop_%s' % suffix)
else:
src_base = os.path.join(my_root, 'train_refer_image')
out_base = os.path.join(my_root, 'train_refer_%s' % suffix)
dlist = []
dir_cnt = 0
if not os.path.exists(out_base): os.makedirs(out_base)
t0 = time.time()
fop.recursive_dir(src_base, dlist, depth=0)
for dir in dlist:
dir_cnt += 1
if file_list is not None:
if dir_cnt < file_list[0] or dir_cnt > file_list[1]: continue
flist = os.listdir(dir)
print(" --- Conv feat: %d: %s frame: %d" %
(dir_cnt, dir, len(flist)))
out_dir = os.path.join(out_base, dir.split('/')[-1])
for f in flist:
file = os.path.join(dir, f)
model.single_infer(file, out_dir)
t1 = time.time()
print("src_base: %s" % src_base)
print("out_base: %s" % out_base)
print("Total time: %d sec" % (t1 - t0))
print("--- extract conv feat End ---")