def _parallel_distance_matrix(fnames, compression_name, pairing_name,
**kwargs):
"""Parallel calculation of distance matrix."""
num_cpus = mp.cpu_count()
manager = mp.Manager()
pool = mp.Pool(num_cpus)
queue = manager.Queue(2 * num_cpus)
sys.stderr.write('Compressing individual files...\n')
progress_bar = ProgressBar(len(fnames))
compression_args = [{
'cname': compression_name,
'fname': fname,
'queue': queue,
'kwargs': kwargs
} for fname in fnames]
async_result = pool.map_async(_parallel_compression_worker,
compression_args)
for _ in xrange(len(fnames)):
queue.get(timeout=5)
progress_bar.increment()
compressed_sizes = async_result.get()
zip_size = dict(zip(fnames, compressed_sizes))
sys.stderr.write('\nCompressing file pairs...\n')
file_pairs = [(fname1, fname2) for fname1 in fnames for fname2 in fnames
if fname1 < fname2]
progress_bar = ProgressBar(len(file_pairs))
ncd_args = [{
'cname': compression_name,
'pname': pairing_name,
'f1': fname1,
'f2': fname2,
'queue': queue,
'zip': (zip_size[fname1], zip_size[fname2]),
'kwargs': kwargs
} for fname1, fname2 in file_pairs]
async_result = pool.map_async(_parallel_ncd_worker, ncd_args)
pool.close()
for _ in xrange(len(file_pairs)):
queue.get(timeout=5)
progress_bar.increment()
ncd_results = async_result.get()
sys.stderr.write('\n')
return ncd_results
def _parallel_distance_matrix(fnames, compression_name, pairing_name, **kwargs):
"""Parallel calculation of distance matrix."""
num_cpus = mp.cpu_count()
manager = mp.Manager()
pool = mp.Pool(num_cpus)
queue = manager.Queue(2*num_cpus)
sys.stderr.write('Compressing individual files...\n')
progress_bar = ProgressBar(len(fnames))
compression_args = [{
'cname': compression_name, 'fname': fname,
'queue': queue, 'kwargs': kwargs}
for fname in fnames]
async_result = pool.map_async(_parallel_compression_worker, compression_args)
for _ in xrange(len(fnames)):
queue.get(timeout=5)
progress_bar.increment()
compressed_sizes = async_result.get()
zip_size = dict(zip(fnames, compressed_sizes))
sys.stderr.write('\nCompressing file pairs...\n')
file_pairs = [(fname1, fname2)
for fname1 in fnames
for fname2 in fnames
if fname1 < fname2]
progress_bar = ProgressBar(len(file_pairs))
ncd_args = [{
'cname': compression_name,
'pname': pairing_name,
'f1': fname1, 'f2': fname2,
'queue': queue, 'zip': (zip_size[fname1], zip_size[fname2]),
'kwargs': kwargs} for fname1, fname2 in file_pairs]
async_result = pool.map_async(_parallel_ncd_worker, ncd_args)
pool.close()
for _ in xrange(len(file_pairs)):
queue.get(timeout=5)
progress_bar.increment()
ncd_results = async_result.get()
sys.stderr.write('\n')
return ncd_results
def _parallel_compress_files(fnames, compression_name, pool=None, queue=None,
verbosity_level=1, **kwargs):
"""Parallel calculation of individual compressed sizes."""
if verbosity_level > 0:
sys.stderr.write('Compressing individual files...\n')
progress_bar = ProgressBar(len(fnames))
pool, queue = _maybe_make_pool_and_queue(pool, queue)
compression_args = [{
'cname': compression_name, 'fname': fname,
'queue': queue, 'kwargs': kwargs}
for fname in fnames]
async_result = pool.map_async(_parallel_compression_worker, compression_args)
for _ in xrange(len(fnames)):
queue.get(timeout=None)
progress_bar.increment() if verbosity_level > 0 else None
compressed_sizes = async_result.get()
sys.stderr.write('\n') if verbosity_level > 0 else None
return dict(zip(fnames, compressed_sizes))
def _parallel_compress_pairs(zip_size, file_pairs, compression_name,
pairing_name, pool=None, queue=None, verbosity_level=1, **kwargs):
"""Parallel calculation of paired file pairs compression."""
if verbosity_level > 0:
sys.stderr.write('Compressing file pairs...\n')
progress_bar = ProgressBar(len(file_pairs))
pool, queue = _maybe_make_pool_and_queue(pool, queue)
ncd_args = [{
'cname': compression_name,
'pname': pairing_name,
'f1': fname1, 'f2': fname2,
'queue': queue, 'zip': (zip_size[fname1], zip_size[fname2]),
'kwargs': kwargs} for fname1, fname2 in file_pairs]
async_result = pool.map_async(_parallel_ncd_worker, ncd_args)
for _ in xrange(len(file_pairs)):
queue.get(timeout=None)
progress_bar.increment() if verbosity_level > 0 else None
ncd_results = async_result.get()
sys.stderr.write('\n') if verbosity_level > 0 else None
return ncd_results
def get_resized_image(url):
return cv2.resize(get_image(url), IMAGE_DIM)
# artist = 'Rebecca Guay'
# artist = None
artist = None if len(sys.argv) < 2 else ' '.join(sys.argv[1:])
path = 'images/' + (artist if artist is not None else "all")
if not os.path.exists(path):
os.mkdir(path)
with open('./data/unique-artwork-20200529172420.json', encoding='utf-8') as f:
data = json.load(f)
bar = ProgressBar(len(data))
for card_obj in data:
if (artist == 'all' or artist is None or
('artist' in card_obj and card_obj['artist'] == artist)
) and 'image_uris' in card_obj and 'art_crop' in card_obj[
'image_uris']:
img = get_resized_image(card_obj['image_uris']['art_crop'])
cv2.imwrite(
path + "/" + card_obj['name'] + ' - ' +
str(int(time.time() * 1000)) + ".png", img)
bar.increment()
