def download_test_set(num_workers, failed_log, compress, verbose, skip, log_file): """ Download the test set. :param num_workers: Number of downloads in parallel. :param failed_log: Where to save failed video ids. :param compress: Decides if the videos should be compressed. :param verbose: Print status. :param skip: Skip classes that already have folders (i.e. at least one video was downloaded). :param log_file: Path to log file for youtube-dl. :return: """ with open(config.TEST_METADATA_PATH) as file: data = json.load(file) pool = parallel.Pool(None, data, config.TEST_ROOT, num_workers, failed_log, compress, verbose, skip, log_file=log_file) pool.start_workers() pool.feed_videos() pool.stop_workers()
def download_classes(classes, num_workers, failed_save_file, compress, verbose, skip, log_file): """ Download all videos of the provided classes. :param classes: List of classes to download. :param num_workers: Number of downloads in parallel. :param failed_save_file: Where to save failed video ids. :param compress: Decides if the videos should be compressed. :param verbose: Print status. :param skip: Skip classes that already have folders (i.e. at least one video was downloaded). :param log_file: Path to log file for youtube-dl. :return: None. """ for list_path, save_root in zip( [config.TRAIN_METADATA_PATH, config.VAL_METADATA_PATH], [config.TRAIN_ROOT, config.VALID_ROOT]): with open(list_path) as file: data = json.load(file) pool = parallel.Pool(classes, data, save_root, num_workers, failed_save_file, compress, verbose, skip, log_file=log_file) pool.start_workers() pool.feed_videos() pool.stop_workers()
def download_classes(classes, num_workers, failed_save_file, compress, verbose, skip, log_file): """ Download all videos of the provided classes. :param classes: List of classes to download. :param num_workers: Number of downloads in parallel. :param failed_save_file: Where to save failed video ids. :param compress: Decides if the videos should be compressed. :param verbose: Print status. :param skip: Skip classes that already have folders (i.e. at least one video was downloaded). :param log_file: Path to log file for youtube-dl. :return: None. """ for list_path, save_root in zip( [config.TRAIN_METADATA_PATH, config.VAL_METADATA_PATH], [config.TRAIN_ROOT, config.VALID_ROOT]): with open(list_path) as file: data = json.load(file) video_ids_all = list(data.keys()) # blob_video = CloudStorage(config.STORAGE_ACCOUNT_NAME, "kinetics700", config.CONNECTION_STRING, config.SAS_TOKEN) # video_stored = [os.path.basename(file_name).replace(".mp4","") for file_name in blob_video.list_blob_names()] blob_image = CloudStorage(config.STORAGE_ACCOUNT_NAME, "kinetics700-image", config.CONNECTION_STRING, config.SAS_TOKEN) image_stored = set([ os.path.basename(file_name).split("_frame")[0] for file_name in blob_image.list_blob_names() ]) data_to_process = { key: val for key, val in data.items() if key not in image_stored } pool = parallel.Pool(classes, data_to_process, save_root, num_workers, failed_save_file, compress, verbose, skip, log_file=log_file) pool.start_workers() pool.feed_videos() pool.stop_workers()
def download_classes_from_file(classes_file, num_workers, failed_save_file, compress, verbose, skip, log_file, stats_file=None): """ Download all videos of the provided classes. :param classes: List of classes to download. :param num_workers: Number of downloads in parallel. :param failed_save_file: Where to save failed video ids. :param compress: Decides if the videos should be compressed. :param verbose: Print status. :param skip: Skip classes that already have folders (i.e. at least one video was downloaded). :param log_file: Path to log file for youtube-dl. :return: None. """ with open(config.SUB_CLASS_PATH) as c_file: classes_data = json.load(c_file) for list_path, save_root in tqdm( zip([config.TEST_METADATA_PATH], [config.TEST_ROOT])): with open(list_path) as file: data = json.load(file) filtered = data # import ipdb; ipdb.set_trace() print('DOWNLOADING {} of {}'.format(len(filtered), save_root)) pool = parallel.Pool(None, data, save_root, num_workers, failed_save_file, compress, verbose, skip, log_file=log_file, stats_file=stats_file) pool.start_workers() pool.feed_videos() pool.stop_workers()
def download_missing(data, save_root, num_workers, failed_save_file, compress, verbose, skip, log_file, stats_file=None): print('DOWNLOADING {} of {}'.format(len(data.keys()), save_root)) pool = parallel.Pool(None, data, save_root, num_workers, failed_save_file, compress, verbose, skip, log_file=log_file, stats_file=stats_file) pool.start_workers() pool.feed_videos() pool.stop_workers()