def download_ext_multi(resources, chunk_size=1024 * 1024, progress_callback=console.progress_bar,
progress_stream=sys.stdout,
progress_template='\r[{counter} of {total}] [{done}{todo}] {name}'):
"""
Download resources, showing a progress bar by default.
Each element should be a `dict` with the url, path and name keys.
Any extra item is passed to :func:`iter_download_to_file` as extra keyword arguments.
"""
for counter, resource in enumerate(sorted(resources, key=lambda r: r['name']), 1):
kwargs, start_time = resource.copy(), time.time()
url, path, name = kwargs.pop('url'), kwargs.pop('path'), kwargs.pop('name')
callback = functools.partial(
progress_callback,
stream=progress_stream,
template=progress_template.format(
counter=counter, done='{done}', name=name, todo='{todo}', total=len(resources)))
if not os.path.exists(path):
makedirs(os.path.dirname(path))
try:
for returned in iter_download_to_file(
url, path, chunk_size=chunk_size, force=False, **kwargs
):
callback(start_time, returned[0], returned[1])
except:
remove(path)
raise
callback(start_time, 1, 1)
progress_stream.write(os.linesep)
def test_chown(tmp_path):
file_a = tmp_path / 'a.txt'
file_b = tmp_path / 'b.txt'
file_c = tmp_path / 'other' / 'c.txt'
filesystem.makedirs(file_c, parent=True)
Path(file_a).touch()
Path(file_b).touch()
Path(file_c).touch()
with mock.patch('os.chown') as chown:
filesystem.chown(file_a, 'root')
chown.assert_called_once_with(file_a, 0, -1)
with mock.patch('os.chown') as chown:
filesystem.chown(tmp_path, 100, 'root')
chown.assert_called_once_with(tmp_path, 100, 0)
with mock.patch('os.chown') as chown:
filesystem.chown(tmp_path, 100, 'root', recursive=True)
chown.assert_has_calls([
mock.call(str(tmp_path), 100, 0),
mock.call(str(file_b), 100, 0),
mock.call(str(file_a), 100, 0),
mock.call(str(file_c.parent), 100, 0),
mock.call(str(file_c), 100, 0),
],
any_order=True)
def small_mp4(request): # pylint:disable=unused-argument
print('Download small.mp4')
filesystem.makedirs(TMP_DIRECTORY)
http.download_ext(SMALL_MP4_URL,
SMALL_MP4_FILENAME,
expected_hash=SMALL_MP4_CHECKSUM,
hash_algorithm='md5',
force=False)
return SMALL_MP4_FILENAME
def download_ext_multi(
resources,
chunk_size=1024 * 1024,
progress_callback=console.progress_bar,
progress_stream=sys.stdout,
progress_template='\r[{counter} of {total}] [{done}{todo}] {name}'):
"""
Download resources, showing a progress bar by default.
Each element should be a `dict` with the url, path and name keys.
Any extra item is passed to :func:`iter_download_to_file` as extra keyword arguments.
"""
for counter, resource in enumerate(
sorted(resources, key=lambda r: r['name']), 1):
kwargs = resource.copy()
start_time = time.time()
url = kwargs.pop('url')
path = kwargs.pop('path')
name = kwargs.pop('name')
callback = functools.partial(progress_callback,
stream=progress_stream,
template=progress_template.format(
counter=counter,
done='{done}',
name=name,
todo='{todo}',
total=len(resources)))
if not os.path.exists(path):
filesystem.makedirs(os.path.dirname(path))
try:
for returned in iter_download_to_file(url,
path,
chunk_size=chunk_size,
force=False,
**kwargs):
callback(start_time, returned[0], returned[1])
except Exception:
filesystem.remove(path)
raise
callback(start_time, 1, 1)
progress_stream.write(os.linesep)
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
epilog='smartcrop.py test bed')
parser.add_argument('scripts', action=FullPaths, nargs='+', type=is_file)
parser.add_argument('-s', '--source', action=FullPaths, type=is_dir)
parser.add_argument('-t', '--target', action=FullPaths, type=is_dir)
parser.add_argument('-p', '--passes', default=3, type=int)
parser.add_argument('-w', '--width', default=200, type=int)
parser.add_argument('-x', '--height', default=200, type=int)
args = parser.parse_args()
if len(set(os.path.basename(s) for s in args.scripts)) < len(args.scripts):
sys.exit('Please make sure scripts names are unique.')
source_filenames = sorted(filesystem.find_recursive(args.source, '*.jpg'))
timing_by_script = collections.defaultdict(list)
for script in args.scripts:
name = os.path.basename(script)
source_directory = args.source + os.path.sep
target_directory = os.path.join(args.target, name) + os.path.sep
for i in range(1, args.passes + 1):
print('Script {0} round {1} of 3'.format(name, i))
start_time = time.time()
for source_filename in source_filenames:
target_filename = source_filename.replace(
source_directory, target_directory)
filesystem.makedirs(target_filename, parent=True)
print(source_filename, target_filename)
assert source_filename != target_filename
subprocess.check_call([
'python',
script,
'--width',
str(args.width),
'--height',
str(args.height),
source_filename,
target_filename,
])
timing_by_script[name].append(time.time() - start_time)
for name, timings in sorted(timing_by_script.items()):
print(name, *('{0:.2f}'.format(t) for t in timings))
def main():
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter,
epilog='smartcrop.py test bed')
parser.add_argument('scripts', action=FullPaths, nargs='+', type=is_file)
parser.add_argument('-s', '--source', action=FullPaths, type=is_dir)
parser.add_argument('-t', '--target', action=FullPaths, type=is_dir)
parser.add_argument('-p', '--passes', default=3, type=int)
parser.add_argument('-w', '--width', default=200, type=int)
parser.add_argument('-x', '--height', default=200, type=int)
args = parser.parse_args()
if len(set(os.path.basename(s) for s in args.scripts)) < len(args.scripts):
sys.exit('Please make sure scripts names are unique.')
source_filenames = sorted(filesystem.find_recursive(args.source, '*.jpg'))
timing_by_script = collections.defaultdict(list)
for script in args.scripts:
name = os.path.basename(script)
source_directory = args.source + os.path.sep
target_directory = os.path.join(args.target, name) + os.path.sep
for i in range(1, args.passes + 1):
print('Script {0} round {1} of 3'.format(name, i))
start_time = time.time()
for source_filename in source_filenames:
target_filename = source_filename.replace(source_directory, target_directory)
filesystem.makedirs(target_filename, parent=True)
print(source_filename, target_filename)
assert source_filename != target_filename
subprocess.check_call([
'python', script,
'--width', str(args.width),
'--height', str(args.height),
source_filename, target_filename,
])
timing_by_script[name].append(time.time() - start_time)
for name, timings in sorted(timing_by_script.items()):
print(name, *('{0:.2f}'.format(t) for t in timings))
def static_ffmpeg(request): # pylint:disable=unused-argument
print('Download ffmpeg static binary')
filesystem.makedirs(TMP_DIRECTORY)
http.download_ext(FFMPEG_RELEASE_URL,
FFMPEG_RELEASE_ARCHIVE,
expected_hash=FFMPEG_RELEASE_CHECKSUM,
hash_algorithm='md5',
force=False)
with tarfile.open(FFMPEG_RELEASE_ARCHIVE) as f:
f.extractall(TMP_DIRECTORY)
class StaticFFprobe(ffmpeg.FFprobe):
executable = FFMPEG_RELEASE_DIRECTORY / 'ffprobe'
class StaticEncodeStatistics(ffmpeg.EncodeStatistics):
ffprobe_class = StaticFFprobe
class StaticFFmpeg(ffmpeg.FFmpeg):
executable = FFMPEG_RELEASE_DIRECTORY / 'ffmpeg'
ffprobe_class = StaticFFprobe
statistics_class = StaticEncodeStatistics
return StaticFFmpeg
def create_directory(self):
if not self.is_pipe:
filesystem.makedirs(self.directory)
def create_directory(self):
if not self.is_pipe:
filesystem.makedirs(self.directory)