def pack_arrays(a1, a2, a3): data = {"random": a1, "sparse": a2, "semisparse": a3} # blosc.set_nthreads(6) # Should be number of *REAL* (not virtual) cores. Can bring speedup sometimes. blosc.set_releasegil(True) # Somehow makes things a bit faster res = msgpack.packb(data, use_bin_type=True) res = blosc.compress(res, cname='zstd', clevel=2) return res
def worms_main(argv): pyrosetta.init("-mute all -beta -preserve_crystinfo --prevent_repacking") blosc.set_releasegil(True) criteria_list, kw = build_worms_setup_from_cli_args(argv) try: worms_main2(criteria_list, kw) except Exception as e: bbdb = kw["db"][0] bbdb.clear() raise e
def test_decompress_releasegil(self): import numpy as np # assume the expected answer was compressed from bytes blosc.set_releasegil(True) expected = b'0123456789' compressed = blosc.compress(expected, typesize=1) # now for all the things that support the buffer interface self.assertEqual(expected, blosc.decompress(compressed)) self.assertEqual(expected, blosc.decompress(memoryview(compressed))) self.assertEqual(expected, blosc.decompress(bytearray(compressed))) self.assertEqual(expected, blosc.decompress(np.array([compressed]))) blosc.set_releasegil(False)
def test_decompress_releasegil(self): import numpy as np # assume the expected answer was compressed from bytes blosc.set_releasegil(True) expected = b'0123456789' compressed = blosc.compress(expected, typesize=1) # now for all the things that support the buffer interface self.assertEqual(expected, blosc.decompress(compressed)) if not PY3X: # Python 3 no longer has the buffer self.assertEqual(expected, blosc.decompress(buffer(compressed))) self.assertEqual(expected, blosc.decompress(memoryview(compressed))) self.assertEqual(expected, blosc.decompress(bytearray(compressed))) self.assertEqual(expected, blosc.decompress(np.array([compressed]))) blosc.set_releasegil(False)
def test_decompress_releasegil(self): import numpy as np # assume the expected answer was compressed from bytes blosc.set_releasegil(True) expected = b'0123456789' compressed = blosc.compress(expected, typesize=1) # now for all the things that support the buffer interface if not PY3X: # Python 3 no longer has the buffer self.assertEqual(expected, blosc.decompress(buffer(compressed))) if not PY26: # memoryview doesn't exist on Python 2.6 self.assertEqual(expected, blosc.decompress(memoryview(compressed))) self.assertEqual(expected, blosc.decompress(bytearray(compressed))) self.assertEqual(expected, blosc.decompress(np.array([compressed]))) blosc.set_releasegil(False)
Includes utilities for determining whether or not to compress """ from contextlib import suppress from functools import partial import logging import random import dask from tlz import identity try: import blosc n = blosc.set_nthreads(2) if hasattr("blosc", "releasegil"): blosc.set_releasegil(True) except ImportError: blosc = False from ..utils import ensure_bytes compressions = {None: {"compress": identity, "decompress": identity}} compressions[False] = compressions[None] # alias default_compression = None logger = logging.getLogger(__name__) with suppress(ImportError): import zlib
def worms_main2(criteria_list, kw): print('worms_main,', len(criteria_list), 'criteria, args:') for k, v in kw.items(): print(' ', k, v) pyrosetta.init('-mute all -beta -preserve_crystinfo') blosc.set_releasegil(True) orig_output_prefix = kw['output_prefix'] for icrit, criteria in enumerate(criteria_list): if len(criteria_list) > 1: assert len(criteria_list) is len(kw['config_file']) name = os.path.basename(kw['config_file'][icrit]) name = name.replace('.config', '') kw['output_prefix'] = orig_output_prefix + '_' + name print('================== start job', icrit, '======================') print('output_prefix:', kw['output_prefix']) print('criteria:', criteria) print('bbspec:', criteria.bbspec) if kw['precache_splices']: print('precaching splices') merge_bblock = kw['merge_bblock'] del kw['merge_bblock'] kw['bbs'] = simple_search_dag(criteria, merge_bblock=None, precache_only=True, **kw) kw['merge_bblock'] = merge_bblock if kw['precache_splices_and_quit']: return global _shared_ssdag if 'bbs' in kw and (len(kw['bbs']) > 2 or kw['bbs'][0] is not kw['bbs'][1]): ############3 # # _shared_ssdag = simple_search_dag( # criteria, print_edge_summary=True, **kw # ) merge_bblock = kw['merge_bblock'] del kw['merge_bblock'] _shared_ssdag = simple_search_dag(criteria, merge_bblock=0, print_edge_summary=True, **kw) kw['merge_bblock'] = merge_bblock print('memuse for global _shared_ssdag:') _shared_ssdag.report_memory_use() #### # if _shared_ssdag: if not 'bbs' in kw: kw['bbs'] = _shared_ssdag.bbs assert len(_shared_ssdag.bbs) == len(kw['bbs']) for a, b in zip(_shared_ssdag.bbs, kw['bbs']): for aa, bb in zip(a, b): assert aa is bb log = worms_main_each_mergebb(criteria, **kw) if kw['pbar']: print('======================== logs ========================') for msg in log: print(msg) print('======================== done ========================')
Includes utilities for determining whether or not to compress """ from __future__ import print_function, division, absolute_import import logging import random import dask from toolz import identity, partial try: import blosc n = blosc.set_nthreads(2) if hasattr('blosc', 'releasegil'): blosc.set_releasegil(True) except ImportError: blosc = False from ..utils import ignoring, ensure_bytes compressions = {None: {'compress': identity, 'decompress': identity}} compressions[False] = compressions[None] # alias default_compression = None