def pack_arrays(a1, a2, a3):
data = {"random": a1, "sparse": a2, "semisparse": a3}
# blosc.set_nthreads(6) # Should be number of *REAL* (not virtual) cores. Can bring speedup sometimes.
blosc.set_releasegil(True) # Somehow makes things a bit faster
res = msgpack.packb(data, use_bin_type=True)
res = blosc.compress(res, cname='zstd', clevel=2)
return res
def worms_main(argv):
pyrosetta.init("-mute all -beta -preserve_crystinfo --prevent_repacking")
blosc.set_releasegil(True)
criteria_list, kw = build_worms_setup_from_cli_args(argv)
try:
worms_main2(criteria_list, kw)
except Exception as e:
bbdb = kw["db"][0]
bbdb.clear()
raise e
def test_decompress_releasegil(self):
import numpy as np
# assume the expected answer was compressed from bytes
blosc.set_releasegil(True)
expected = b'0123456789'
compressed = blosc.compress(expected, typesize=1)
# now for all the things that support the buffer interface
self.assertEqual(expected, blosc.decompress(compressed))
self.assertEqual(expected, blosc.decompress(memoryview(compressed)))
self.assertEqual(expected, blosc.decompress(bytearray(compressed)))
self.assertEqual(expected, blosc.decompress(np.array([compressed])))
blosc.set_releasegil(False)
def test_decompress_releasegil(self):
import numpy as np
# assume the expected answer was compressed from bytes
blosc.set_releasegil(True)
expected = b'0123456789'
compressed = blosc.compress(expected, typesize=1)
# now for all the things that support the buffer interface
self.assertEqual(expected, blosc.decompress(compressed))
if not PY3X:
# Python 3 no longer has the buffer
self.assertEqual(expected, blosc.decompress(buffer(compressed)))
self.assertEqual(expected, blosc.decompress(memoryview(compressed)))
self.assertEqual(expected, blosc.decompress(bytearray(compressed)))
self.assertEqual(expected, blosc.decompress(np.array([compressed])))
blosc.set_releasegil(False)
def test_decompress_releasegil(self):
import numpy as np
# assume the expected answer was compressed from bytes
blosc.set_releasegil(True)
expected = b'0123456789'
compressed = blosc.compress(expected, typesize=1)
# now for all the things that support the buffer interface
if not PY3X:
# Python 3 no longer has the buffer
self.assertEqual(expected, blosc.decompress(buffer(compressed)))
if not PY26:
# memoryview doesn't exist on Python 2.6
self.assertEqual(expected,
blosc.decompress(memoryview(compressed)))
self.assertEqual(expected, blosc.decompress(bytearray(compressed)))
self.assertEqual(expected, blosc.decompress(np.array([compressed])))
blosc.set_releasegil(False)
Includes utilities for determining whether or not to compress
"""
from contextlib import suppress
from functools import partial
import logging
import random
import dask
from tlz import identity
try:
import blosc
n = blosc.set_nthreads(2)
if hasattr("blosc", "releasegil"):
blosc.set_releasegil(True)
except ImportError:
blosc = False
from ..utils import ensure_bytes
compressions = {None: {"compress": identity, "decompress": identity}}
compressions[False] = compressions[None] # alias
default_compression = None
logger = logging.getLogger(__name__)
with suppress(ImportError):
import zlib
def worms_main2(criteria_list, kw):
print('worms_main,', len(criteria_list), 'criteria, args:')
for k, v in kw.items():
print(' ', k, v)
pyrosetta.init('-mute all -beta -preserve_crystinfo')
blosc.set_releasegil(True)
orig_output_prefix = kw['output_prefix']
for icrit, criteria in enumerate(criteria_list):
if len(criteria_list) > 1:
assert len(criteria_list) is len(kw['config_file'])
name = os.path.basename(kw['config_file'][icrit])
name = name.replace('.config', '')
kw['output_prefix'] = orig_output_prefix + '_' + name
print('================== start job', icrit, '======================')
print('output_prefix:', kw['output_prefix'])
print('criteria:', criteria)
print('bbspec:', criteria.bbspec)
if kw['precache_splices']:
print('precaching splices')
merge_bblock = kw['merge_bblock']
del kw['merge_bblock']
kw['bbs'] = simple_search_dag(criteria,
merge_bblock=None,
precache_only=True,
**kw)
kw['merge_bblock'] = merge_bblock
if kw['precache_splices_and_quit']:
return
global _shared_ssdag
if 'bbs' in kw and (len(kw['bbs']) > 2
or kw['bbs'][0] is not kw['bbs'][1]):
############3
#
# _shared_ssdag = simple_search_dag(
# criteria, print_edge_summary=True, **kw
# )
merge_bblock = kw['merge_bblock']
del kw['merge_bblock']
_shared_ssdag = simple_search_dag(criteria,
merge_bblock=0,
print_edge_summary=True,
**kw)
kw['merge_bblock'] = merge_bblock
print('memuse for global _shared_ssdag:')
_shared_ssdag.report_memory_use()
####
#
if _shared_ssdag:
if not 'bbs' in kw:
kw['bbs'] = _shared_ssdag.bbs
assert len(_shared_ssdag.bbs) == len(kw['bbs'])
for a, b in zip(_shared_ssdag.bbs, kw['bbs']):
for aa, bb in zip(a, b):
assert aa is bb
log = worms_main_each_mergebb(criteria, **kw)
if kw['pbar']:
print('======================== logs ========================')
for msg in log:
print(msg)
print('======================== done ========================')
Includes utilities for determining whether or not to compress
"""
from __future__ import print_function, division, absolute_import
import logging
import random
import dask
from toolz import identity, partial
try:
import blosc
n = blosc.set_nthreads(2)
if hasattr('blosc', 'releasegil'):
blosc.set_releasegil(True)
except ImportError:
blosc = False
from ..utils import ignoring, ensure_bytes
compressions = {None: {'compress': identity,
'decompress': identity}}
compressions[False] = compressions[None] # alias
default_compression = None