def load_identities(path, patterns=('*.jpg', '*.jpeg'), **kwargs): identities = [] for name in os.listdir(path): for filename in filesystem.find_recursive(os.path.join(path, name), patterns, **kwargs): identities.append(Identity(name, filename)) return np.array(identities)
def run(self): from pytoolbox import encoding, filesystem from pytoolbox.subprocess import cmd encoding.configure_unicode() docs_directory = os.path.join(os.path.dirname(__file__), 'docs') source_directory = os.path.join(docs_directory, 'source') package_directory = os.path.join(os.path.dirname(__file__), 'pytoolbox') # Cleanup previously generated restructured files for path in filesystem.find_recursive(source_directory, r'^pytoolbox.*\.rst$', unix_wildcards=False): os.remove(path) cmd([ 'sphinx-apidoc', '--force', '--module-first', '--separate', '-o', source_directory, package_directory ]) shutil.rmtree(os.path.join(docs_directory, 'build', 'html'), ignore_errors=True) result = cmd('make html', cwd=docs_directory, fail=False) print( '{0}Outputs{0}======={0}{1}{0}{0}Errors{0}======{0}{2}{0}'.format( os.linesep, result['stdout'].decode('utf-8'), result['stderr'].decode('utf-8'))) sys.exit(1 if result['stderr'] else 0)
def run(self): # pylint:disable=no-self-use from pytoolbox import filesystem project = Path(__file__).resolve().parent source = project / 'docs' / 'source' # Cleanup previously generated restructured files for path in filesystem.find_recursive(source, r'^pytoolbox.*\.rst$', regex=True): filesystem.remove(path) subprocess.run([ 'sphinx-apidoc', '--force', '--module-first', '--separate', '-o', source, project / 'pytoolbox' ], check=True) filesystem.remove(project / 'docs' / 'build' / 'html', recursive=True) subprocess.run(['make', 'html'], cwd=project / 'docs', check=True)
def main(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, epilog='smartcrop.py test bed') parser.add_argument('scripts', action=FullPaths, nargs='+', type=is_file) parser.add_argument('-s', '--source', action=FullPaths, type=is_dir) parser.add_argument('-t', '--target', action=FullPaths, type=is_dir) parser.add_argument('-p', '--passes', default=3, type=int) parser.add_argument('-w', '--width', default=200, type=int) parser.add_argument('-x', '--height', default=200, type=int) args = parser.parse_args() if len(set(os.path.basename(s) for s in args.scripts)) < len(args.scripts): sys.exit('Please make sure scripts names are unique.') source_filenames = sorted(filesystem.find_recursive(args.source, '*.jpg')) timing_by_script = collections.defaultdict(list) for script in args.scripts: name = os.path.basename(script) source_directory = args.source + os.path.sep target_directory = os.path.join(args.target, name) + os.path.sep for i in range(1, args.passes + 1): print('Script {0} round {1} of 3'.format(name, i)) start_time = time.time() for source_filename in source_filenames: target_filename = source_filename.replace( source_directory, target_directory) filesystem.makedirs(target_filename, parent=True) print(source_filename, target_filename) assert source_filename != target_filename subprocess.check_call([ 'python', script, '--width', str(args.width), '--height', str(args.height), source_filename, target_filename, ]) timing_by_script[name].append(time.time() - start_time) for name, timings in sorted(timing_by_script.items()): print(name, *('{0:.2f}'.format(t) for t in timings))
def run(self): from pytoolbox import encoding, filesystem from pytoolbox.subprocess import cmd encoding.configure_unicode() docs_directory = os.path.join(os.path.dirname(__file__), 'docs') source_directory = os.path.join(docs_directory, 'source') package_directory = os.path.join(os.path.dirname(__file__), 'pytoolbox') # Cleanup previously generated restructured files for path in filesystem.find_recursive(source_directory, r'^pytoolbox.*\.rst$', unix_wildcards=False): os.remove(path) cmd(['sphinx-apidoc', '--force', '--module-first', '--separate', '-o', source_directory, package_directory]) shutil.rmtree(os.path.join(docs_directory, 'build', 'html'), ignore_errors=True) result = cmd('make html', cwd=docs_directory, fail=False) print('{0}Outputs{0}======={0}{1}{0}{0}Errors{0}======{0}{2}{0}'.format( os.linesep, result['stdout'].decode('utf-8'), result['stderr'].decode('utf-8')) ) sys.exit(1 if result['stderr'] else 0)
def run(self): from pytoolbox import encoding, filesystem from pytoolbox.subprocess import cmd encoding.configure_unicode() docs_directory = os.path.join(os.path.dirname(__file__), 'docs') source_directory = os.path.join(docs_directory, 'source') package_directory = os.path.join(os.path.dirname(__file__), 'pytoolbox') # Cleanup previously generated restructured files for path in filesystem.find_recursive( source_directory, r'^pytoolbox.*\.rst$', unix_wildcards=False ): os.remove(path) cmd([ 'sphinx-apidoc', '--force', '--module-first', '--separate', '-o', source_directory, package_directory ]) shutil.rmtree(os.path.join(docs_directory, 'build', 'html'), ignore_errors=True) subprocess.check_call(['make', 'html'], cwd=docs_directory)
def main(): parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter, epilog='smartcrop.py test bed') parser.add_argument('scripts', action=FullPaths, nargs='+', type=is_file) parser.add_argument('-s', '--source', action=FullPaths, type=is_dir) parser.add_argument('-t', '--target', action=FullPaths, type=is_dir) parser.add_argument('-p', '--passes', default=3, type=int) parser.add_argument('-w', '--width', default=200, type=int) parser.add_argument('-x', '--height', default=200, type=int) args = parser.parse_args() if len(set(os.path.basename(s) for s in args.scripts)) < len(args.scripts): sys.exit('Please make sure scripts names are unique.') source_filenames = sorted(filesystem.find_recursive(args.source, '*.jpg')) timing_by_script = collections.defaultdict(list) for script in args.scripts: name = os.path.basename(script) source_directory = args.source + os.path.sep target_directory = os.path.join(args.target, name) + os.path.sep for i in range(1, args.passes + 1): print('Script {0} round {1} of 3'.format(name, i)) start_time = time.time() for source_filename in source_filenames: target_filename = source_filename.replace(source_directory, target_directory) filesystem.makedirs(target_filename, parent=True) print(source_filename, target_filename) assert source_filename != target_filename subprocess.check_call([ 'python', script, '--width', str(args.width), '--height', str(args.height), source_filename, target_filename, ]) timing_by_script[name].append(time.time() - start_time) for name, timings in sorted(timing_by_script.items()): print(name, *('{0:.2f}'.format(t) for t in timings))
def run(self): from pytoolbox import encoding, filesystem from pytoolbox.subprocess import cmd encoding.configure_unicode() docs_directory = os.path.join(os.path.dirname(__file__), 'docs') source_directory = os.path.join(docs_directory, 'source') package_directory = os.path.join(os.path.dirname(__file__), 'pytoolbox') # Cleanup previously generated restructured files for path in filesystem.find_recursive(source_directory, r'^pytoolbox.*\.rst$', unix_wildcards=False): os.remove(path) cmd([ 'sphinx-apidoc', '--force', '--module-first', '--separate', '-o', source_directory, package_directory ]) shutil.rmtree(os.path.join(docs_directory, 'build', 'html'), ignore_errors=True) subprocess.check_call(['make', 'html'], cwd=docs_directory)
def main(): signal.signal(signal.SIGINT, lambda *args: sys.exit(0)) log = setup_log() parser = argparse.ArgumentParser(epilog='Archive stuff on S3.') parser.add_argument('--config', action=FullPaths, required=True, type=is_file) parser.add_argument('--simulate', action='store_true') parser.add_argument('--verbosity', choices=(0, 1, 2), default=0, type=int) args = parser.parse_args() def log_it(verbosity, level, message, **extra): if args.verbosity >= verbosity: extra['level'] = LEVEL_MAP[level] getattr(log, level)(message, extra=extra) s3 = boto3.client('s3') log_it(1, 'info', 'Process started') try: with open(args.config) as config_file: config = yaml.load(config_file) if config['enabled']: log_it(1, 'info', 'Its time to transfer!') if args.simulate: log_it(1, 'warning', 'Simulation mode enabled') for transfer in config['transfers']: name = transfer['name'] log_it(1, 'info', 'Handling transfer', transfer=name) bucket = transfer['bucket'] delete = transfer['delete'] directory = transfer['directory'] prefix = transfer['prefix'].format(host_fqdn=socket.getfqdn()) processed_bytes = processed_count = skipped_bytes = skipped_count = 0 for source_path in filesystem.find_recursive( directory, transfer['patterns'], unix_wildcards=False ): target_path = os.path.join(prefix, os.path.relpath(source_path, directory)) target_obj = aws.s3.load_object_meta(s3, bucket, target_path, fail=False) with open(source_path, 'rb') as source_file: # Retrieve metadata from source and target source_size = filesystem.get_size(source_path) target_size = None if target_obj is None else target_obj['ContentLength'] target_md5 = None if target_obj is None else target_obj['ETag'].strip('"') source_md5 = crypto.checksum( source_path, is_path=True, algorithm='md5', chunk_size=1024 * 1024) changed = source_md5 != target_md5 log_it( 2, 'info', 'File', transfer=name, changed=changed, source_md5=source_md5, source_path=source_path, source_size=source_size, target_md5=target_md5, target_path=target_path, target_size=target_size) if changed: processed_bytes += source_size processed_count += 1 else: skipped_bytes += source_size skipped_count += 1 if not args.simulate: aws.s3.write_object(s3, bucket, target_path, source_file) if delete: filesystem.remove(source_path) log_it( 1, 'info', 'Summary', transfer=name, processed_bytes=processed_bytes, processed_count=processed_count, skipped_bytes=skipped_bytes, skipped_count=skipped_count) else: log.warning('Process is disabled') except Exception as e: log.exception(e) finally: log_it(1, 'info', 'Process ended')
def check(directory): for filename in sorted(filesystem.find_recursive(directory, '*.yml')): print(filename) with open(filename) as f: yaml.load(f)