def load_identities(path, patterns=('*.jpg', '*.jpeg'), **kwargs):
identities = []
for name in os.listdir(path):
for filename in filesystem.find_recursive(os.path.join(path, name),
patterns, **kwargs):
identities.append(Identity(name, filename))
return np.array(identities)
def run(self):
from pytoolbox import encoding, filesystem
from pytoolbox.subprocess import cmd
encoding.configure_unicode()
docs_directory = os.path.join(os.path.dirname(__file__), 'docs')
source_directory = os.path.join(docs_directory, 'source')
package_directory = os.path.join(os.path.dirname(__file__),
'pytoolbox')
# Cleanup previously generated restructured files
for path in filesystem.find_recursive(source_directory,
r'^pytoolbox.*\.rst$',
unix_wildcards=False):
os.remove(path)
cmd([
'sphinx-apidoc', '--force', '--module-first', '--separate', '-o',
source_directory, package_directory
])
shutil.rmtree(os.path.join(docs_directory, 'build', 'html'),
ignore_errors=True)
result = cmd('make html', cwd=docs_directory, fail=False)
print(
'{0}Outputs{0}======={0}{1}{0}{0}Errors{0}======{0}{2}{0}'.format(
os.linesep, result['stdout'].decode('utf-8'),
result['stderr'].decode('utf-8')))
sys.exit(1 if result['stderr'] else 0)
def run(self): # pylint:disable=no-self-use
from pytoolbox import filesystem
project = Path(__file__).resolve().parent
source = project / 'docs' / 'source'
# Cleanup previously generated restructured files
for path in filesystem.find_recursive(source,
r'^pytoolbox.*\.rst$',
regex=True):
filesystem.remove(path)
subprocess.run([
'sphinx-apidoc', '--force', '--module-first', '--separate', '-o',
source, project / 'pytoolbox'
],
check=True)
filesystem.remove(project / 'docs' / 'build' / 'html', recursive=True)
subprocess.run(['make', 'html'], cwd=project / 'docs', check=True)
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
epilog='smartcrop.py test bed')
parser.add_argument('scripts', action=FullPaths, nargs='+', type=is_file)
parser.add_argument('-s', '--source', action=FullPaths, type=is_dir)
parser.add_argument('-t', '--target', action=FullPaths, type=is_dir)
parser.add_argument('-p', '--passes', default=3, type=int)
parser.add_argument('-w', '--width', default=200, type=int)
parser.add_argument('-x', '--height', default=200, type=int)
args = parser.parse_args()
if len(set(os.path.basename(s) for s in args.scripts)) < len(args.scripts):
sys.exit('Please make sure scripts names are unique.')
source_filenames = sorted(filesystem.find_recursive(args.source, '*.jpg'))
timing_by_script = collections.defaultdict(list)
for script in args.scripts:
name = os.path.basename(script)
source_directory = args.source + os.path.sep
target_directory = os.path.join(args.target, name) + os.path.sep
for i in range(1, args.passes + 1):
print('Script {0} round {1} of 3'.format(name, i))
start_time = time.time()
for source_filename in source_filenames:
target_filename = source_filename.replace(
source_directory, target_directory)
filesystem.makedirs(target_filename, parent=True)
print(source_filename, target_filename)
assert source_filename != target_filename
subprocess.check_call([
'python',
script,
'--width',
str(args.width),
'--height',
str(args.height),
source_filename,
target_filename,
])
timing_by_script[name].append(time.time() - start_time)
for name, timings in sorted(timing_by_script.items()):
print(name, *('{0:.2f}'.format(t) for t in timings))
def run(self):
from pytoolbox import encoding, filesystem
from pytoolbox.subprocess import cmd
encoding.configure_unicode()
docs_directory = os.path.join(os.path.dirname(__file__), 'docs')
source_directory = os.path.join(docs_directory, 'source')
package_directory = os.path.join(os.path.dirname(__file__), 'pytoolbox')
# Cleanup previously generated restructured files
for path in filesystem.find_recursive(source_directory, r'^pytoolbox.*\.rst$', unix_wildcards=False):
os.remove(path)
cmd(['sphinx-apidoc', '--force', '--module-first', '--separate', '-o', source_directory, package_directory])
shutil.rmtree(os.path.join(docs_directory, 'build', 'html'), ignore_errors=True)
result = cmd('make html', cwd=docs_directory, fail=False)
print('{0}Outputs{0}======={0}{1}{0}{0}Errors{0}======{0}{2}{0}'.format(
os.linesep, result['stdout'].decode('utf-8'), result['stderr'].decode('utf-8'))
)
sys.exit(1 if result['stderr'] else 0)
def run(self):
from pytoolbox import encoding, filesystem
from pytoolbox.subprocess import cmd
encoding.configure_unicode()
docs_directory = os.path.join(os.path.dirname(__file__), 'docs')
source_directory = os.path.join(docs_directory, 'source')
package_directory = os.path.join(os.path.dirname(__file__), 'pytoolbox')
# Cleanup previously generated restructured files
for path in filesystem.find_recursive(
source_directory, r'^pytoolbox.*\.rst$', unix_wildcards=False
):
os.remove(path)
cmd([
'sphinx-apidoc', '--force', '--module-first', '--separate', '-o',
source_directory, package_directory
])
shutil.rmtree(os.path.join(docs_directory, 'build', 'html'), ignore_errors=True)
subprocess.check_call(['make', 'html'], cwd=docs_directory)
def main():
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter,
epilog='smartcrop.py test bed')
parser.add_argument('scripts', action=FullPaths, nargs='+', type=is_file)
parser.add_argument('-s', '--source', action=FullPaths, type=is_dir)
parser.add_argument('-t', '--target', action=FullPaths, type=is_dir)
parser.add_argument('-p', '--passes', default=3, type=int)
parser.add_argument('-w', '--width', default=200, type=int)
parser.add_argument('-x', '--height', default=200, type=int)
args = parser.parse_args()
if len(set(os.path.basename(s) for s in args.scripts)) < len(args.scripts):
sys.exit('Please make sure scripts names are unique.')
source_filenames = sorted(filesystem.find_recursive(args.source, '*.jpg'))
timing_by_script = collections.defaultdict(list)
for script in args.scripts:
name = os.path.basename(script)
source_directory = args.source + os.path.sep
target_directory = os.path.join(args.target, name) + os.path.sep
for i in range(1, args.passes + 1):
print('Script {0} round {1} of 3'.format(name, i))
start_time = time.time()
for source_filename in source_filenames:
target_filename = source_filename.replace(source_directory, target_directory)
filesystem.makedirs(target_filename, parent=True)
print(source_filename, target_filename)
assert source_filename != target_filename
subprocess.check_call([
'python', script,
'--width', str(args.width),
'--height', str(args.height),
source_filename, target_filename,
])
timing_by_script[name].append(time.time() - start_time)
for name, timings in sorted(timing_by_script.items()):
print(name, *('{0:.2f}'.format(t) for t in timings))
def run(self):
from pytoolbox import encoding, filesystem
from pytoolbox.subprocess import cmd
encoding.configure_unicode()
docs_directory = os.path.join(os.path.dirname(__file__), 'docs')
source_directory = os.path.join(docs_directory, 'source')
package_directory = os.path.join(os.path.dirname(__file__),
'pytoolbox')
# Cleanup previously generated restructured files
for path in filesystem.find_recursive(source_directory,
r'^pytoolbox.*\.rst$',
unix_wildcards=False):
os.remove(path)
cmd([
'sphinx-apidoc', '--force', '--module-first', '--separate', '-o',
source_directory, package_directory
])
shutil.rmtree(os.path.join(docs_directory, 'build', 'html'),
ignore_errors=True)
subprocess.check_call(['make', 'html'], cwd=docs_directory)
def main():
signal.signal(signal.SIGINT, lambda *args: sys.exit(0))
log = setup_log()
parser = argparse.ArgumentParser(epilog='Archive stuff on S3.')
parser.add_argument('--config', action=FullPaths, required=True, type=is_file)
parser.add_argument('--simulate', action='store_true')
parser.add_argument('--verbosity', choices=(0, 1, 2), default=0, type=int)
args = parser.parse_args()
def log_it(verbosity, level, message, **extra):
if args.verbosity >= verbosity:
extra['level'] = LEVEL_MAP[level]
getattr(log, level)(message, extra=extra)
s3 = boto3.client('s3')
log_it(1, 'info', 'Process started')
try:
with open(args.config) as config_file:
config = yaml.load(config_file)
if config['enabled']:
log_it(1, 'info', 'Its time to transfer!')
if args.simulate:
log_it(1, 'warning', 'Simulation mode enabled')
for transfer in config['transfers']:
name = transfer['name']
log_it(1, 'info', 'Handling transfer', transfer=name)
bucket = transfer['bucket']
delete = transfer['delete']
directory = transfer['directory']
prefix = transfer['prefix'].format(host_fqdn=socket.getfqdn())
processed_bytes = processed_count = skipped_bytes = skipped_count = 0
for source_path in filesystem.find_recursive(
directory,
transfer['patterns'],
unix_wildcards=False
):
target_path = os.path.join(prefix, os.path.relpath(source_path, directory))
target_obj = aws.s3.load_object_meta(s3, bucket, target_path, fail=False)
with open(source_path, 'rb') as source_file:
# Retrieve metadata from source and target
source_size = filesystem.get_size(source_path)
target_size = None if target_obj is None else target_obj['ContentLength']
target_md5 = None if target_obj is None else target_obj['ETag'].strip('"')
source_md5 = crypto.checksum(
source_path,
is_path=True,
algorithm='md5',
chunk_size=1024 * 1024)
changed = source_md5 != target_md5
log_it(
2, 'info', 'File',
transfer=name,
changed=changed,
source_md5=source_md5,
source_path=source_path,
source_size=source_size,
target_md5=target_md5,
target_path=target_path,
target_size=target_size)
if changed:
processed_bytes += source_size
processed_count += 1
else:
skipped_bytes += source_size
skipped_count += 1
if not args.simulate:
aws.s3.write_object(s3, bucket, target_path, source_file)
if delete:
filesystem.remove(source_path)
log_it(
1, 'info', 'Summary',
transfer=name,
processed_bytes=processed_bytes,
processed_count=processed_count,
skipped_bytes=skipped_bytes,
skipped_count=skipped_count)
else:
log.warning('Process is disabled')
except Exception as e:
log.exception(e)
finally:
log_it(1, 'info', 'Process ended')
def check(directory):
for filename in sorted(filesystem.find_recursive(directory, '*.yml')):
print(filename)
with open(filename) as f:
yaml.load(f)