def bbox_worker(workers_bbox, workers_returncode, cachedir=None):
""" The worker.
Parameters
----------
workers_bbox, workers_returncode: multiprocessing.Queue
the input and output queues.
cachedir: string
the directory in which the smart-caching will work.
"""
from casper.lib.cache import Memory
import traceback
mem = Memory(cachedir)
while True:
inputs = workers_bbox.get()
if inputs == FLAG_ALL_DONE:
workers_returncode.put(FLAG_WORKER_FINISHED_PROCESSING)
break
try:
process_name, box_funcdesc, bbox_inputs = inputs
bbox = mem.cache(Bbox(box_funcdesc))
for control_name, value in bbox_inputs.items():
setattr(bbox.inputs, control_name, value)
bbox_returncode = bbox(process_name)
bbox_returncode[process_name]["exitcode"] = 0
except:
bbox_returncode = {process_name: {}}
bbox_returncode[process_name]["inputs"] = {}
bbox_returncode[process_name]["outputs"] = {}
bbox_returncode[process_name]["exitcode"] = (
"1 - {0}'".format(traceback.format_exc()))
workers_returncode.put(bbox_returncode)
class TestMemory(unittest.TestCase):
""" Execute a bbox using smart-caching functionalities.
"""
def setUp(self):
""" Initialize the TestMemory class.
"""
self.myfuncdesc = "casper.demo.module.a_function_to_wrap"
self.mycloth = "casper.demo.module.clothing"
self.mypipedesc = "casper.demo.pipeline.xml"
self.myclothingdesc = "casper.demo.clothing_pipeline.xml"
self.mypipexmldesc = "casper.demo.xml_pipeline.xml"
self.mypyramiddesc = "casper.demo.pyramid_pipeline.xml"
self.myswitchdesc = "casper.demo.switch_pipeline.xml"
self.myiterativedesc = "casper.demo.iterative_pipeline.xml"
self.myfile = os.path.abspath(__file__)
self.mydir = os.path.dirname(self.myfile)
self.cachedir = tempfile.mkdtemp()
self.mem = Memory(self.cachedir)
def tearDown(self):
""" Destroy the temporary directory.
"""
if self.cachedir is not None:
shutil.rmtree(self.cachedir)
def test_proxy_box_with_copy(self):
""" Test the proxy box behaviours with cache and copy.
"""
# Return to new line
print
# Create the box
mybbox = Bbox(self.myfuncdesc)
# Call the box in the cache
self.mem.clear()
cached_box = self.mem.cache(mybbox)
returncode = cached_box()
self.assertEqual(cached_box.outputs.fname.value, None)
self.assertEqual(returncode[mybbox.id]["outputs"]["fname"], None)
self.assertEqual(cached_box.outputs.string.value, "None-None")
self.assertEqual(
returncode[mybbox.id]["outputs"]["string"], "None-None")
returncode = cached_box()
# Test the copy in the cache
cached_box.inputs.fname = self.myfile
cached_box.inputs.directory = self.mydir
returncode = cached_box()
self.assertEqual(cached_box.outputs.fname.value, self.myfile)
self.assertEqual(
returncode[mybbox.id]["outputs"]["fname"], self.myfile)
self.assertEqual(cached_box.outputs.string.value,
"-".join([repr(self.myfile), repr(self.mydir)]))
self.assertEqual(returncode[mybbox.id]["outputs"]["string"],
"-".join([repr(self.myfile), repr(self.mydir)]))
cache_dir, _, _ = cached_box._get_box_id()
self.assertFalse(os.path.isfile(
os.path.join(cache_dir, os.path.basename(self.myfile))))
self.mem.clear()
cached_box.outputs.fname.copy = True
returncode = cached_box()
self.assertTrue(os.path.isfile(
os.path.join(cache_dir, os.path.basename(self.myfile))))
def test_proxy_box_without_cache(self):
""" Test the proxy box behaviours without cache.
"""
# Return to new line
print
# Create the empty cache
self.cachedir = None
self.mem = Memory(self.cachedir)
# Create the box
mybbox = Bbox(self.mycloth)
cached_box = self.mem.cache(mybbox)
# Test parameters
self.assertEqual(id(mybbox), id(cached_box.box))
self.assertTrue(isinstance(repr(cached_box), str))
# Test creation with memorized object
cached_box_2 = self.mem.cache(cached_box)
self.assertEqual(id(mybbox), id(cached_box_2.box))
# Call the box in the cache
returncode = cached_box()
self.assertEqual(cached_box.outputs.outp.value, None)
self.assertEqual(returncode[cached_box.id]["outputs"]["outp"], None)
cached_box.inputs.inp = "slip"
returncode = cached_box()
self.assertEqual(cached_box.outputs.outp.value, "slip")
self.assertEqual(returncode[mybbox.id]["outputs"]["outp"], "slip")
returncode = cached_box(inp="pantalon")
self.assertEqual(cached_box.outputs.outp.value, "pantalon")
self.assertEqual(returncode[mybbox.id]["outputs"]["outp"], "pantalon")
def test_proxy_box_with_cache(self):
""" Test the proxy box behaviours with cache.
"""
# Return to new line
print
# Create the box
mybbox = Bbox(self.mycloth)
cached_box = self.mem.cache(mybbox)
# Test parameters
self.assertEqual(id(mybbox), id(cached_box.box))
self.assertTrue(isinstance(repr(cached_box), str))
# Call the box in the cache
returncode = cached_box()
self.assertEqual(cached_box.outputs.outp.value, None)
self.assertEqual(returncode[mybbox.id]["outputs"]["outp"], None)
returncode = cached_box()
cached_box.inputs.inp = "slip"
returncode = cached_box()
self.assertEqual(cached_box.outputs.outp.value, "slip")
self.assertEqual(returncode[mybbox.id]["outputs"]["outp"], "slip")
returncode = cached_box()
returncode = cached_box(inp="pantalon")
self.assertEqual(cached_box.outputs.outp.value, "pantalon")
self.assertEqual(returncode[mybbox.id]["outputs"]["outp"], "pantalon")
def test_memorized_box(self):
""" Test the memorized box creation.
"""
# Create the box
mybbox = Bbox(self.mycloth)
# Test raises
self.assertRaises(ValueError, MemorizedBox, mybbox, None)
self.assertRaises(ValueError, MemorizedBox, mybbox, "")
# Test parameters
mbox = MemorizedBox(mybbox, self.cachedir)
self.assertTrue(mbox.timestamp is not None)
# Test copy
file_mapping = []
python_object = {
"1": [self.myfile, 156],
"2": (self.mydir, )
}
mbox._copy_files_to_memory(python_object, self.cachedir, file_mapping)
self.assertTrue(os.path.isfile(
os.path.join(self.cachedir, os.path.basename(self.myfile))))
# Test hash
hash1, _ = mbox._get_argument_hash()
mbox.inputs.inp.nohash = True
hash2, _ = mbox._get_argument_hash()
self.assertEqual(hash1, hash2)
mbox.inputs.inp = self.myfile
hash3, _ = mbox._get_argument_hash()
self.assertEqual(hash2, hash3)
mbox.inputs.inp.nohash = False
hash4, _ = mbox._get_argument_hash()
self.assertTrue(hash3 != hash4)
# Test fingerprint
updated_object = mbox._add_fingerprints(python_object)
self.assertTrue(isinstance(updated_object["1"][0], dict))
# Test nohash search
control = List(content="Int", nohash=False)
has_nohash = has_attribute(control, "noshash", True, recursive=True)
self.assertFalse(has_nohash)
control.inner_control.nohash = True
has_nohash = has_attribute(control, "noshash", True, recursive=True)
self.assertTrue(has_nohash)
# Test encoder
updated_object = tuple_json_encoder(python_object)
self.assertTrue(isinstance(updated_object["2"], dict))