return parser.parse_args()
def get_filename(path):
"""Return the filename of a path
Args: path: path to file
Returns:
filename: name of file (without extension)
"""
return os.path.splitext(os.path.basename(path))[0]
if __name__ == '__main__':
init_console_logger(LOGGER, verbose=True)
args = parse_arguments()
video_files = glob.glob(os.path.join(args.video_data_dir, "*.mp4"))
video_files += glob.glob(os.path.join(args.video_data_dir, "*/*.mp4"))
audio_files = glob.glob(os.path.join(args.audio_data_dir, "*.wav"))
print("num of video files {}".format(len(video_files)))
print("num of audio files {}".format(len(audio_files)))
video_file_names = set([get_filename(path) for path in video_files])
audio_file_names = set([get_filename(path) for path in audio_files])
valid_file_names = video_file_names & audio_file_names
print("num of valid files {}".format(len(valid_file_names)))
parser.add_argument('output_dir',
action='store',
type=str,
help='Path to directory where output files will be stored')
parser.add_argument('filename_prefix',
action='store',
type=str,
help='Path to directory where output files will be stored')
return parser.parse_args()
if __name__ == '__main__':
init_console_logger(LOGGER, verbose=True)
args = parse_arguments()
train_list, valid_list, test_list \
= get_subset_split(args.data_dir,
valid_ratio=args.valid_ratio,
test_ratio=args.test_ratio,
random_state=args.random_state,
metadata_path=args.metadata_path,
filter_path=args.filter_path,
ontology_path=args.ontology_path)
output_dir = args.output_dir
filename_prefix = args.filename_prefix
train_subset_path = os.path.join(output_dir, filename_prefix + '_train.csv')
help='Path to subset file')
parser.add_argument('num_samples',
action='store',
type=int,
help='(Minimum) number of samples to generate')
parser.add_argument(
'output_dir',
action='store',
type=str,
help='Path to directory where output files will be stored')
args = parser.parse_args()
init_console_logger(LOGGER, verbose=args.verbose)
multiprocessing_logging.install_mp_handler()
# Just round up for now
num_workers = args.num_workers
batch_size = args.batch_size
batches_per_worker = int(
math.ceil(args.num_samples / (num_workers * batch_size)))
worker_func = partial(sample_and_save,
subset_path=args.subset_path,
num_batches=batches_per_worker,
output_dir=args.output_dir,
num_streamers=args.num_streamers,
batch_size=batch_size,
random_state=args.random_state,
def download_audioset(data_dir,
ffmpeg_path,
ffprobe_path,
eval_segments_path,
balanced_train_segments_path,
unbalanced_train_segments_path,
disable_logging=False,
verbose=False,
num_workers=4,
log_path=None,
**ffmpeg_cfg):
"""
Download AudioSet files
Args:
data_dir: Directory where dataset files will
be saved
(Type: str)
ffmpeg_path: Path to ffmpeg executable
(Type: str)
ffprobe_path: Path to ffprobe executable
(Type: str)
eval_segments_path: Path to evaluation segments file
(Type: str)
balanced_train_segments_path: Path to balanced train segments file
(Type: str)
unbalanced_train_segments_path: Path to unbalanced train segments file
(Type: str)
Keyword Args:
disable_logging: Disables logging to a file if True
(Type: bool)
verbose: Prints verbose information to stdout
if True
(Type: bool)
num_workers: Number of multiprocessing workers used
to download videos
(Type: int)
log_path: Path where log file will be saved. If
None, saved to './audiosetdl.log'
(Type: str or None)
**ffmpeg_cfg: Configuration for audio and video
downloading and decoding done by ffmpeg
(Type: dict[str, *])
"""
init_console_logger(LOGGER, verbose=verbose)
if not disable_logging:
init_file_logger(LOGGER, log_path=log_path)
multiprocessing_logging.install_mp_handler()
LOGGER.debug('Initialized logging.')
download_subset(eval_segments_path, data_dir, ffmpeg_path, ffprobe_path,
num_workers, **ffmpeg_cfg)
download_subset(balanced_train_segments_path, data_dir, ffmpeg_path,
ffprobe_path, num_workers, **ffmpeg_cfg)
download_subset(unbalanced_train_segments_path, data_dir, ffmpeg_path,
ffprobe_path, num_workers, **ffmpeg_cfg)
outfile.write(words)
def parse_arguments():
"""
Get command line arguments
"""
parser = argparse.ArgumentParser(
description='Preprocess text data into lists of tokens')
parser.add_argument('dataset_dir',
help='Path to directory containing text files',
type=str)
parser.add_argument('output_dir',
help='Path to output directory',
type=str)
parser.add_argument('target_ext',
help='Extension of relevant text files',
type=str)
parser.add_argument('-n',
'--num-workers',
dest='num_workers',
type=int,
default=1,
help='Number of workers')
return vars(parser.parse_args())
if __name__ == '__main__':
init_console_logger(LOGGER)
preprocess_dataset(**(parse_arguments()))
type=str,
help='Path to subset file')
parser.add_argument('num_samples',
action='store',
type=int,
help='(Minimum) number of samples to generate')
parser.add_argument('output_dir',
action='store',
type=str,
help='Path to directory where output files will be stored')
args = parser.parse_args()
init_console_logger(LOGGER, verbose=args.verbose)
multiprocessing_logging.install_mp_handler()
# Just round up for now
num_workers = args.num_workers
batch_size = args.batch_size
batches_per_worker = int(math.ceil(args.num_samples / (num_workers * batch_size)))
worker_func = partial(sample_and_save,
subset_path=args.subset_path,
num_batches=batches_per_worker,
output_dir=args.output_dir,
num_streamers=args.num_streamers,
batch_size=batch_size,
random_state=args.random_state,
precompute=args.precompute,
parser.add_argument('-v', '--verbose', dest='verbose', action='store_true')
parser.add_argument('audio_dir',
type=str,
help='Path to directory containing audio files')
parser.add_argument(
'output_dir',
type=str,
help='Path to directory where model files will be output')
args = parser.parse_args()
return vars(args)
if __name__ == '__main__':
args = parse_arguments()
init_console_logger(LOGGER, args['verbose'])
LOGGER.info('Initialized logger.')
batch_size = args['batch_size']
latent_dim = args['latent_dim']
ngpus = args['ngpus']
model_size = args['model_size']
model_dir = os.path.join(args['output_dir'],
datetime.datetime.now().strftime("%Y%m%d%H%M%S"))
args['model_dir'] = model_dir
# Create output directory
if not os.path.exists(model_dir):
os.makedirs(model_dir)