def run(self, src, distances_path):
folder_path = os.path.dirname(distances_path)
try:
t.create_folder(folder_path)
distances = self.compute_dist(src)
if distances is None:
return None
t.save_array(distances, distances_path)
return distances_path
except Exception as e:
print(folder_path, e)
return None
def run(self, src, vector_path):
folder_path = os.path.dirname(vector_path)
# try:
t.create_folder(folder_path)
# tmp_video_path = os.path.join(self.workdir, dst, os.path.basename(src))
# shutil.copy(src, tmp_video_path)
video_vector = self.compute_vector(src)
t.save_array(video_vector, vector_path)
# return vector_path
# except Exception as e:
# print(folder_path, e)
# return None
return vector_path
def process(self, youtube_path, exports_path, workdir):
yt_list = t.load_json(youtube_path)
ex_list = t.load_json(exports_path)
self.test_len = 0
self.run_from = 41609
self.timestamp = time.time()
self.list_to_process = yt_list + ex_list
self.err_file = os.path.join(workdir, 'errors.json')
run_to = self.run_from + self.test_len if self.test_len != 0 else len(
self.list_to_process)
errors = []
for idx in tqdm(range(run_to),
ascii=True,
desc='process {} links'.format(run_to)):
if idx < self.run_from:
continue
record = self.list_to_process[idx]
path, out_path_rel = record['src'], record['dst']
out_path = t.create_folder(os.path.join(workdir, out_path_rel))
name = 'image.png'
name_info = 'info.json'
out = os.path.join(out_path, name)
out_info = os.path.join(out_path, name_info)
if os.path.isfile(out):
continue
try:
# print(f_idx)
frame, info = self.get_full_frame(path)
info['src'] = path
# np.save(out, frame)
cv2.imwrite(out, frame)
with open(out_info, 'w') as outfile:
json.dump(info, outfile, indent=4)
except Exception as e:
err = '{}: {}'.format(path, e)
print(err)
errors.append(err)
# print(info)
if len(errors):
with open(self.err_file, 'w') as outfile:
json.dump(errors, outfile, indent=4)
workdir = '/mnt/data/palpatine/DATASETS/YT_LINK/workdir'
#
# exports_list_path = '/mnt/data/palpatine/SAMPLES/YT_LINK/sample/sources_a.json'
# yt_list_path = '/mnt/data/palpatine/SAMPLES/YT_LINK/sample/sources_b.json'
# workdir = '/mnt/data/palpatine/SAMPLES/YT_LINK/sample/workdir'
# exports_list = '/mnt/data/palpatine/SAMPLES/YT_LINK/REF/list_exports_test.json'
# yt_list = '/mnt/data/palpatine/SAMPLES/YT_LINK/REF/list_youtube_test.json'
# exports_list = '/mnt/data/palpatine/SAMPLES/YT_LINK/REF/list_exports.json'
# yt_list = '/mnt/data/palpatine/SAMPLES/YT_LINK/REF/list_youtube.json'
#
# workdir = '/mnt/data/palpatine/SAMPLES/YT_LINK/REF/workdir'
test_len = 0
ss = 0
DIFF_LIMIT = 1000
workdir = t.create_folder(workdir)
t_start = time.time()
print('\nCREATE PROCESS POOL AND IMAGES')
vl = CreateProcessPool(youtube_path=yt_list_path, exports_path=exports_list_path, workdir=workdir, test_len=test_len, ss=ss)
youtube_to_process_path, export_to_process_path = vl.run()
# CreateImages().process(youtube_to_process_path, export_to_process_path, workdir)
print('\nMATCH BY DURATION')
m = MatchByDuration(youtube_path=youtube_to_process_path, export_path=export_to_process_path, workdir=workdir, diff_limit=DIFF_LIMIT)
m.get_duration_lists()
print('done in {:.02f} s'.format(time.time() - t_start))