def _parallel_distance_matrix(fnames, compression_name, pairing_name, **kwargs): """Parallel calculation of distance matrix.""" num_cpus = mp.cpu_count() manager = mp.Manager() pool = mp.Pool(num_cpus) queue = manager.Queue(2 * num_cpus) sys.stderr.write('Compressing individual files...\n') progress_bar = ProgressBar(len(fnames)) compression_args = [{ 'cname': compression_name, 'fname': fname, 'queue': queue, 'kwargs': kwargs } for fname in fnames] async_result = pool.map_async(_parallel_compression_worker, compression_args) for _ in xrange(len(fnames)): queue.get(timeout=5) progress_bar.increment() compressed_sizes = async_result.get() zip_size = dict(zip(fnames, compressed_sizes)) sys.stderr.write('\nCompressing file pairs...\n') file_pairs = [(fname1, fname2) for fname1 in fnames for fname2 in fnames if fname1 < fname2] progress_bar = ProgressBar(len(file_pairs)) ncd_args = [{ 'cname': compression_name, 'pname': pairing_name, 'f1': fname1, 'f2': fname2, 'queue': queue, 'zip': (zip_size[fname1], zip_size[fname2]), 'kwargs': kwargs } for fname1, fname2 in file_pairs] async_result = pool.map_async(_parallel_ncd_worker, ncd_args) pool.close() for _ in xrange(len(file_pairs)): queue.get(timeout=5) progress_bar.increment() ncd_results = async_result.get() sys.stderr.write('\n') return ncd_results
def _parallel_distance_matrix(fnames, compression_name, pairing_name, **kwargs): """Parallel calculation of distance matrix.""" num_cpus = mp.cpu_count() manager = mp.Manager() pool = mp.Pool(num_cpus) queue = manager.Queue(2*num_cpus) sys.stderr.write('Compressing individual files...\n') progress_bar = ProgressBar(len(fnames)) compression_args = [{ 'cname': compression_name, 'fname': fname, 'queue': queue, 'kwargs': kwargs} for fname in fnames] async_result = pool.map_async(_parallel_compression_worker, compression_args) for _ in xrange(len(fnames)): queue.get(timeout=5) progress_bar.increment() compressed_sizes = async_result.get() zip_size = dict(zip(fnames, compressed_sizes)) sys.stderr.write('\nCompressing file pairs...\n') file_pairs = [(fname1, fname2) for fname1 in fnames for fname2 in fnames if fname1 < fname2] progress_bar = ProgressBar(len(file_pairs)) ncd_args = [{ 'cname': compression_name, 'pname': pairing_name, 'f1': fname1, 'f2': fname2, 'queue': queue, 'zip': (zip_size[fname1], zip_size[fname2]), 'kwargs': kwargs} for fname1, fname2 in file_pairs] async_result = pool.map_async(_parallel_ncd_worker, ncd_args) pool.close() for _ in xrange(len(file_pairs)): queue.get(timeout=5) progress_bar.increment() ncd_results = async_result.get() sys.stderr.write('\n') return ncd_results
def _parallel_compress_files(fnames, compression_name, pool=None, queue=None, verbosity_level=1, **kwargs): """Parallel calculation of individual compressed sizes.""" if verbosity_level > 0: sys.stderr.write('Compressing individual files...\n') progress_bar = ProgressBar(len(fnames)) pool, queue = _maybe_make_pool_and_queue(pool, queue) compression_args = [{ 'cname': compression_name, 'fname': fname, 'queue': queue, 'kwargs': kwargs} for fname in fnames] async_result = pool.map_async(_parallel_compression_worker, compression_args) for _ in xrange(len(fnames)): queue.get(timeout=None) progress_bar.increment() if verbosity_level > 0 else None compressed_sizes = async_result.get() sys.stderr.write('\n') if verbosity_level > 0 else None return dict(zip(fnames, compressed_sizes))
def _parallel_compress_pairs(zip_size, file_pairs, compression_name, pairing_name, pool=None, queue=None, verbosity_level=1, **kwargs): """Parallel calculation of paired file pairs compression.""" if verbosity_level > 0: sys.stderr.write('Compressing file pairs...\n') progress_bar = ProgressBar(len(file_pairs)) pool, queue = _maybe_make_pool_and_queue(pool, queue) ncd_args = [{ 'cname': compression_name, 'pname': pairing_name, 'f1': fname1, 'f2': fname2, 'queue': queue, 'zip': (zip_size[fname1], zip_size[fname2]), 'kwargs': kwargs} for fname1, fname2 in file_pairs] async_result = pool.map_async(_parallel_ncd_worker, ncd_args) for _ in xrange(len(file_pairs)): queue.get(timeout=None) progress_bar.increment() if verbosity_level > 0 else None ncd_results = async_result.get() sys.stderr.write('\n') if verbosity_level > 0 else None return ncd_results
def get_resized_image(url): return cv2.resize(get_image(url), IMAGE_DIM) # artist = 'Rebecca Guay' # artist = None artist = None if len(sys.argv) < 2 else ' '.join(sys.argv[1:]) path = 'images/' + (artist if artist is not None else "all") if not os.path.exists(path): os.mkdir(path) with open('./data/unique-artwork-20200529172420.json', encoding='utf-8') as f: data = json.load(f) bar = ProgressBar(len(data)) for card_obj in data: if (artist == 'all' or artist is None or ('artist' in card_obj and card_obj['artist'] == artist) ) and 'image_uris' in card_obj and 'art_crop' in card_obj[ 'image_uris']: img = get_resized_image(card_obj['image_uris']['art_crop']) cv2.imwrite( path + "/" + card_obj['name'] + ' - ' + str(int(time.time() * 1000)) + ".png", img) bar.increment()