def test_get(self):
"""test get method"""
rec = DotDict(self.rec1)
expect = [1, 2]
result = rec.get('a.c')
self.assertEqual(expect, result)
self.assertEqual(expect, rec['a.c'])
def test_get(self):
"""test get method"""
rec = DotDict(self.rec1)
expect = [1, 2]
result = rec.get("a.c")
self.assertEqual(expect, result)
self.assertEqual(expect, rec["a.c"])
def test_DotDict(self):
"""Test DotDict class"""
res = {u"zip": {u"code": u"14850"}}
mdict = DotDict(res)
mdict["zip.code"] = 14850
expect = {u"zip": {u"code": 14850}}
self.assertEqual(expect, mdict)
res = {"a": {"b": {"c": 10}, "d": 10}}
mdict = DotDict(res)
mdict["x.y.z"] = 10
expect = {"a": {"b": {"c": 10}, "d": 10}, "x": {"y": {"z": 10}}}
self.assertEqual(expect, mdict)
mdict["a.b.k.m"] = 10
expect = {"a": {"b": {"c": 10, "k": {"m": 10}}, "d": 10}, "x": {"y": {"z": 10}}}
self.assertEqual(expect, mdict)
expect = 10
result = mdict.get("a.b.k.m")
self.assertEqual(expect, result)
res = {"a": {"b": {"c": 10}, "d": [{"x": 1}, {"x": 2}]}}
mdict = DotDict(res)
expect = 1
result = mdict.get("a.d.x")
self.assertEqual(expect, result)
expect = None
result = mdict.get("a.M.Z")
self.assertEqual(expect, result)
res = {"a": {"b": {"c": 1, "d": 2}}}
mdict = DotDict(res)
expect = {"a": {"b": {"c": 1}}}
mdict.delete("a.b.d")
self.assertEqual(expect, mdict)
def test_get_keys(self):
"""test get_keys method"""
rec = DotDict(self.rec1)
expect = ["a", "a.b", "a.c"]
expect.sort()
result = rec.get_keys()
result.sort()
self.assertEqual(expect, result)
rec = DotDict(self.rec2)
result = rec.get_keys()
result.sort()
self.assertEqual(expect, result)
def test_DotDict_keys(self):
"""Test DotDict get_keys method"""
res = {"a": [{"b": 1, "c": 1}, {"c": 2}]}
mdict = DotDict(res)
expect = ["a.b", "a.c"]
result = [r for r in mdict.get_keys("a")]
self.assertEqual(set(expect), set(result))
res = {"a": [{"b": [{"c": 2}, {"c": {"d": 1}}]}, {"b": [{"c": 4}, {"c": 5}]}]}
mdict = DotDict(res)
expect = ["a.b", "a.b.c", "a.b.c.d"]
result = [r for r in mdict.get_keys("a")]
self.assertEqual(set(expect), set(result))
def test_DotDict_list(self):
"""Test DotDict class"""
res = {"a": [{"b": 1, "c": 1}, {"c": 1}]}
mdict = DotDict(res)
expect = 1
result = mdict.get("a.b")
self.assertEqual(expect, result)
res = {"a": [{"c": 1}, {"b": 1, "c": 1}]}
mdict = DotDict(res)
expect = 1
result = mdict.get("a.b")
self.assertEqual(expect, result)
def __init__(self, profiling=False, schedfixed=False, deviceCPU=False):
self._profiling = profiling
self._schedfixed = schedfixed
self._deviceCPU = deviceCPU
self.pm = PluginManager()
self.graph = TaskGraph()
self.tasks = DotDict()
def test_DotDict_values(self):
"""Test DotDict get_values method"""
res = {"a": [{"b": 1, "c": 1}, {"c": 2}]}
mdict = DotDict(res)
expect = [1]
result = [r for r in mdict.get_values("a.b")]
self.assertEqual(expect, result)
expect = [1, 2]
result = [r for r in mdict.get_values("a.c")]
self.assertEqual(expect, result)
res = {"a": [{"b": [{"c": 2}, {"c": 3}]}, {"b": [{"c": 4}, {"c": 5}]}]}
mdict = DotDict(res)
expect = [2, 3, 4, 5]
result = [r for r in mdict.get_values("a.b.c")]
self.assertEqual(expect, result)
def __init__(self, parms={}):
super(PivJob, self).__init__(profiling=False, schedfixed=True, deviceCPU=False)
self.parms = DotDict(default_parms)
self.parms.update(parms)
self.parms.ring_number = \
(self.parms.ring_end - self.parms.ring_start) / self.parms.ring_step + 1
self.setup_basic_tasks()
def test_DotDict_keys(self):
"""Test DotDict get_keys method"""
res = {'a': [{'b': 1, 'c': 1}, {'c': 2}]}
mdict = DotDict(res)
expect = ['a.b', 'a.c']
result = [r for r in mdict.get_keys('a')]
self.assertEqual(set(expect), set(result))
res = {
'a': [{
'b': [{
'c': 2
}, {
'c': {
'd': 1
}
}]
}, {
'b': [{
'c': 4
}, {
'c': 5
}]
}]
}
mdict = DotDict(res)
expect = ['a.b', 'a.b.c', 'a.b.c.d']
result = [r for r in mdict.get_keys('a')]
self.assertEqual(set(expect), set(result))
def test_get_keys(self):
"""test get_keys method"""
rec = DotDict(self.rec1)
expect = ['a', 'a.b', 'a.c']
expect.sort()
result = rec.get_keys()
result.sort()
self.assertEqual(expect, result)
rec = DotDict(self.rec2)
result = rec.get_keys()
result.sort()
self.assertEqual(expect, result)
def test_DotDict_list(self):
"""Test DotDict class"""
res = {'a': [{'b': 1, 'c': 1}, {'c': 1}]}
mdict = DotDict(res)
expect = 1
result = mdict.get('a.b')
self.assertEqual(expect, result)
res = {'a': [{'c': 1}, {'b': 1, 'c': 1}]}
mdict = DotDict(res)
expect = 1
result = mdict.get('a.b')
self.assertEqual(expect, result)
def test_DotDict_values(self):
"""Test DotDict get_values method"""
res = {'a': [{'b': 1, 'c': 1}, {'c': 2}]}
mdict = DotDict(res)
expect = [1]
result = [r for r in mdict.get_values('a.b')]
self.assertEqual(expect, result)
expect = [1, 2]
result = [r for r in mdict.get_values('a.c')]
self.assertEqual(expect, result)
res = {'a': [{'b': [{'c': 2}, {'c': 3}]}, {'b': [{'c': 4}, {'c': 5}]}]}
mdict = DotDict(res)
expect = [2, 3, 4, 5]
result = [r for r in mdict.get_values('a.b.c')]
self.assertEqual(expect, result)
def test_set(self):
"""test set method"""
rec = DotDict(self.rec1)
rec['a.d'] = 111
self.assertEqual(rec['a.d'], 111)
self.assertRaises(Exception, rec.set, ('a.c.d', 1))
def test_get_values(self):
"""test get_values method"""
rec = DotDict(self.rec2)
expect = [1, 2, 10, 20]
result = [o for o in rec.get_values('a.c')]
self.assertEqual(expect, result)
class UfoJob(UfoJob, LogMixin):
def __init__(self, profiling=False, schedfixed=False, deviceCPU=False):
self._profiling = profiling
self._schedfixed = schedfixed
self._deviceCPU = deviceCPU
self.pm = PluginManager()
self.graph = TaskGraph()
self.tasks = DotDict()
def init_scheduler(self):
if self._deviceCPU:
self.logger.debug('run on cpu')
self.resource = Ufo.Resources(device_type=Ufo.DeviceType.CPU)
else:
self.logger.debug('run on gpu')
self.resource = Ufo.Resources(device_type=Ufo.DeviceType.GPU)
if self._schedfixed:
self.logger.debug('fixed Scheduler')
self.sched = Ufo.FixedScheduler()
else:
self.logger.debug('normal Scheduler')
self.sched = Ufo.Scheduler()
self.sched.set_resources(self.resource)
self.sched.set_property('enable_tracing', self._profiling)
def run_t(self, n=1):
self.init_scheduler()
def timer_function():
self.sched.run(self.graph)
tm = Timer(timer_function)
t = tm.timeit(n)
# print self.sched
# print self.resource
return t
@property
def profiling(self):
return self._profiling
@profiling.setter
def profiling(self, value):
self._profiling = value
@property
def schedfixed(self):
return self._schedfixed
@schedfixed.setter
def schedfixed(self, value):
self._schedfixed = value
@property
def deviceCPU(self):
return self._deviceCPU
@deviceCPU.setter
def deviceCPU(self, value):
self._deviceCPU = value
def branch(self, *args):
try:
b = [self.tasks.get(n) for n in args]
self.graph.connect_branch(b)
except KeyError:
sys.exit("task %s does not exists" % n)
return b
def add_copy_task(self, taskname):
task = Ufo.CopyTask()
self.tasks[taskname] = task
def add_task(self, taskname, plugin=None, **kargs):
if self.tasks.get('taskname'):
print 'Warning: %s already exists' % taskname
if plugin is None:
plugin = taskname
task = self.pm.get_task(plugin, **kargs)
self.tasks[taskname] = task
def test_delete(self):
"""test delete method"""
rec = DotDict(self.rec1)
rec.delete('a.c')
expect = {'a': {'b': 1}}
self.assertEqual(expect, rec)
def test_delete(self):
"""test delete method"""
rec = DotDict(self.rec1)
rec.delete("a.c")
expect = {"a": {"b": 1}}
self.assertEqual(expect, rec)
def test_get_values(self):
"""test get_values method"""
rec = DotDict(self.rec2)
expect = [1, 2, 10, 20]
result = [o for o in rec.get_values("a.c")]
self.assertEqual(expect, result)
def test_DotDict(self):
"""Test DotDict class"""
res = {u'zip': {u'code': u'14850'}}
mdict = DotDict(res)
mdict['zip.code'] = 14850
expect = {u'zip': {u'code': 14850}}
self.assertEqual(expect, mdict)
res = {'a': {'b': {'c': 10}, 'd': 10}}
mdict = DotDict(res)
mdict['x.y.z'] = 10
expect = {'a': {'b': {'c': 10}, 'd': 10}, 'x': {'y': {'z': 10}}}
self.assertEqual(expect, mdict)
mdict['a.b.k.m'] = 10
expect = {
'a': {
'b': {
'c': 10,
'k': {
'm': 10
}
},
'd': 10
},
'x': {
'y': {
'z': 10
}
}
}
self.assertEqual(expect, mdict)
expect = 10
result = mdict.get('a.b.k.m')
self.assertEqual(expect, result)
res = {'a': {'b': {'c': 10}, 'd': [{'x': 1}, {'x': 2}]}}
mdict = DotDict(res)
expect = 1
result = mdict.get('a.d.x')
self.assertEqual(expect, result)
expect = None
result = mdict.get('a.M.Z')
self.assertEqual(expect, result)
res = {'a': {'b': {'c': 1, 'd': 2}}}
mdict = DotDict(res)
expect = {'a': {'b': {'c': 1}}}
mdict.delete('a.b.d')
self.assertEqual(expect, mdict)
class PivJob(UfoJob):
def __init__(self, parms={}):
super(PivJob, self).__init__(profiling=False, schedfixed=True, deviceCPU=False)
self.parms = DotDict(default_parms)
self.parms.update(parms)
self.parms.ring_number = \
(self.parms.ring_end - self.parms.ring_start) / self.parms.ring_step + 1
self.setup_basic_tasks()
def setup_basic_tasks(self):
p = self.parms
sc = self.parms.scale
self.add_task('crop', x=p.xshift, y=p.yshift, width=1024, height=1024)
self.add_task('contrast', 'piv_contrast')
self.add_task('rescale', factor=1.0/sc)
self.add_task('input_fft', 'fft', dimensions=2)
self.add_task('ring_fft', 'fft', dimensions=2)
self.add_task('ring_stack', 'stack', number=p.ring_number)
self.add_task('ring_loop', 'loop', count=p.number)
self.add_task('ring_convolution', 'complex_mult')
self.add_task('ring_slice', 'slice')
self.add_task('ring_pattern',
start=p.ring_start/sc, end=p.ring_end/sc, step=p.ring_step/sc,
thickness=p.ring_thickness/sc, method=p.ring_method,
width=1024/sc, height=1024/sc)
self.add_task('hessian_kernel', sigma=2.0/sc, width=1024/sc, height=1024/sc)
self.add_task('hessian_fft', 'fft', dimensions=2)
self.add_task('hessian_loop', 'loop', count=p.number*p.ring_number)
self.add_task('hessian_convolution', 'complex_mult')
self.add_task('hessian_analysis')
self.add_copy_task('hessian_broadcast')
self.add_task('ifft', dimensions=2)
self.add_task('local_maxima', sigma=p.maxima_sigma)
self.add_task('label_cluster', 'label-cluster')
self.add_task('combine_test', 'combine-test')
self.add_task('blob_test', alpha=p.blob_alpha)
self.add_task('sum')
self.add_task('ring_writer')
p = self.parms
scale = p.scale
self.add_task('read', path=p.in_path, number=p.number, start=p.start)
self.add_task('write', filename=p.out_file)
self.add_task('stack1', 'stack', number=p.ring_number)
self.add_task('stack2', 'stack', number=p.ring_number)
self.add_task('stack3', 'stack', number=p.ring_number)
self.add_task('unstack1', 'slice')
self.add_task('unstack2', 'slice')
self.add_task('m1', 'monitor')
self.add_task('m2', 'monitor')
self.add_task('m3', 'monitor')
self.add_task('m4', 'monitor')
self.add_task('m5', 'monitor')
self.add_task('m6', 'monitor')
self.add_task('m7', 'monitor')
self.add_task('null')
def setup_tasks(self):
pass
def setup_graph(self, flag):
if flag == 1:
b1 = self.branch('read', 'crop', 'rescale', 'contrast', 'input_fft')#, 'm1')
b2 = self.branch('ring_pattern', 'ring_stack', 'ring_fft', 'ring_loop')#, 'm2')
b3 = self.branch('ring_convolution', 'ifft', 'write')
self.graph.merge_branch(b1, b2, b3)
elif flag == 2:
b1 = self.branch('read', 'crop', 'rescale', 'contrast', 'input_fft')
b2 = self.branch('ring_pattern', 'ring_stack', 'ring_fft', 'ring_loop')
b3 = self.branch('ring_convolution', 'ring_slice')
self.graph.merge_branch(b1, b2, b3)
b4 = self.branch('hessian_kernel', 'hessian_fft', 'hessian_loop')
b5 = self.branch('hessian_convolution', 'ifft', 'hessian_analysis', 'write')
self.graph.merge_branch(b3, b4, b5)
else:
b1 = self.branch('read', 'crop', 'rescale', 'contrast', 'input_fft')
b2 = self.branch('ring_pattern', 'ring_stack', 'ring_fft', 'ring_loop')
b3 = self.branch('ring_convolution', 'ring_slice')
self.graph.merge_branch(b1, b2, b3)
b4 = self.branch('hessian_kernel', 'hessian_fft', 'hessian_loop')
b5 = self.branch('hessian_convolution', 'ifft', 'hessian_analysis',
'hessian_broadcast', 'stack1', 'unstack1')
self.graph.merge_branch(b3, b4, b5)
b6 = self.branch('hessian_broadcast', 'local_maxima', 'label_cluster',
'stack2', 'combine_test', 'unstack2')
b7 = self.branch('blob_test', 'stack3', 'sum', 'write')
self.graph.merge_branch(b5, b6, b7)
def run(self, flag=None):
self.setup_tasks()
self.setup_graph(flag)
self.log_tasks()
runtime = self.run_t()
self.logger.info('')
self.logger.info('Program finished in %s seconds' % runtime )
def log_tasks(self):
scale = self.tasks.rescale.props.factor
fmt_f = "%16s.%-16s= %8.3f"
fmt_i = "%16s.%-16s= %8i"
self.logger.info( 'input: ' + self.tasks.read.props.path)
self.logger.info( 'output: ' + self.tasks.write.props.filename)
self.logger.info( '')
self.logger.info( fmt_i % ('Ring', 'start', self.tasks.ring_pattern.props.start/scale))
self.logger.info( fmt_i % ('Ring', 'end', self.tasks.ring_pattern.props.end/scale))
self.logger.info( fmt_i % ('Ring', 'step', self.tasks.ring_pattern.props.step/scale))
self.logger.info( fmt_f % ('Contrast', 'low_cut', self.tasks.contrast.props.c1))
self.logger.info( fmt_f % ('Contrast', 'high_cut', self.tasks.contrast.props.c2))
self.logger.info( fmt_f % ('Contrast', 'shift', self.tasks.contrast.props.c3))
self.logger.info( fmt_f % ('Contrast', 'width', self.tasks.contrast.props.c4))
self.logger.info( fmt_f % ('Contrast', 'gamma', self.tasks.contrast.props.gamma))
self.logger.info( fmt_f % ('LocalMax', 'threshold', self.tasks.local_maxima.props.sigma))
self.logger.info( fmt_f % ('BlobTest', 'alpha', self.tasks.blob_test.props.alpha))
