def test_get(self): """test get method""" rec = DotDict(self.rec1) expect = [1, 2] result = rec.get('a.c') self.assertEqual(expect, result) self.assertEqual(expect, rec['a.c'])
def test_get(self): """test get method""" rec = DotDict(self.rec1) expect = [1, 2] result = rec.get("a.c") self.assertEqual(expect, result) self.assertEqual(expect, rec["a.c"])
def test_DotDict(self): """Test DotDict class""" res = {u"zip": {u"code": u"14850"}} mdict = DotDict(res) mdict["zip.code"] = 14850 expect = {u"zip": {u"code": 14850}} self.assertEqual(expect, mdict) res = {"a": {"b": {"c": 10}, "d": 10}} mdict = DotDict(res) mdict["x.y.z"] = 10 expect = {"a": {"b": {"c": 10}, "d": 10}, "x": {"y": {"z": 10}}} self.assertEqual(expect, mdict) mdict["a.b.k.m"] = 10 expect = {"a": {"b": {"c": 10, "k": {"m": 10}}, "d": 10}, "x": {"y": {"z": 10}}} self.assertEqual(expect, mdict) expect = 10 result = mdict.get("a.b.k.m") self.assertEqual(expect, result) res = {"a": {"b": {"c": 10}, "d": [{"x": 1}, {"x": 2}]}} mdict = DotDict(res) expect = 1 result = mdict.get("a.d.x") self.assertEqual(expect, result) expect = None result = mdict.get("a.M.Z") self.assertEqual(expect, result) res = {"a": {"b": {"c": 1, "d": 2}}} mdict = DotDict(res) expect = {"a": {"b": {"c": 1}}} mdict.delete("a.b.d") self.assertEqual(expect, mdict)
def test_get_keys(self): """test get_keys method""" rec = DotDict(self.rec1) expect = ["a", "a.b", "a.c"] expect.sort() result = rec.get_keys() result.sort() self.assertEqual(expect, result) rec = DotDict(self.rec2) result = rec.get_keys() result.sort() self.assertEqual(expect, result)
def test_DotDict_keys(self): """Test DotDict get_keys method""" res = {"a": [{"b": 1, "c": 1}, {"c": 2}]} mdict = DotDict(res) expect = ["a.b", "a.c"] result = [r for r in mdict.get_keys("a")] self.assertEqual(set(expect), set(result)) res = {"a": [{"b": [{"c": 2}, {"c": {"d": 1}}]}, {"b": [{"c": 4}, {"c": 5}]}]} mdict = DotDict(res) expect = ["a.b", "a.b.c", "a.b.c.d"] result = [r for r in mdict.get_keys("a")] self.assertEqual(set(expect), set(result))
def test_DotDict_list(self): """Test DotDict class""" res = {"a": [{"b": 1, "c": 1}, {"c": 1}]} mdict = DotDict(res) expect = 1 result = mdict.get("a.b") self.assertEqual(expect, result) res = {"a": [{"c": 1}, {"b": 1, "c": 1}]} mdict = DotDict(res) expect = 1 result = mdict.get("a.b") self.assertEqual(expect, result)
def __init__(self, profiling=False, schedfixed=False, deviceCPU=False): self._profiling = profiling self._schedfixed = schedfixed self._deviceCPU = deviceCPU self.pm = PluginManager() self.graph = TaskGraph() self.tasks = DotDict()
def test_DotDict_values(self): """Test DotDict get_values method""" res = {"a": [{"b": 1, "c": 1}, {"c": 2}]} mdict = DotDict(res) expect = [1] result = [r for r in mdict.get_values("a.b")] self.assertEqual(expect, result) expect = [1, 2] result = [r for r in mdict.get_values("a.c")] self.assertEqual(expect, result) res = {"a": [{"b": [{"c": 2}, {"c": 3}]}, {"b": [{"c": 4}, {"c": 5}]}]} mdict = DotDict(res) expect = [2, 3, 4, 5] result = [r for r in mdict.get_values("a.b.c")] self.assertEqual(expect, result)
def __init__(self, parms={}): super(PivJob, self).__init__(profiling=False, schedfixed=True, deviceCPU=False) self.parms = DotDict(default_parms) self.parms.update(parms) self.parms.ring_number = \ (self.parms.ring_end - self.parms.ring_start) / self.parms.ring_step + 1 self.setup_basic_tasks()
def test_DotDict_keys(self): """Test DotDict get_keys method""" res = {'a': [{'b': 1, 'c': 1}, {'c': 2}]} mdict = DotDict(res) expect = ['a.b', 'a.c'] result = [r for r in mdict.get_keys('a')] self.assertEqual(set(expect), set(result)) res = { 'a': [{ 'b': [{ 'c': 2 }, { 'c': { 'd': 1 } }] }, { 'b': [{ 'c': 4 }, { 'c': 5 }] }] } mdict = DotDict(res) expect = ['a.b', 'a.b.c', 'a.b.c.d'] result = [r for r in mdict.get_keys('a')] self.assertEqual(set(expect), set(result))
def test_get_keys(self): """test get_keys method""" rec = DotDict(self.rec1) expect = ['a', 'a.b', 'a.c'] expect.sort() result = rec.get_keys() result.sort() self.assertEqual(expect, result) rec = DotDict(self.rec2) result = rec.get_keys() result.sort() self.assertEqual(expect, result)
def test_DotDict_list(self): """Test DotDict class""" res = {'a': [{'b': 1, 'c': 1}, {'c': 1}]} mdict = DotDict(res) expect = 1 result = mdict.get('a.b') self.assertEqual(expect, result) res = {'a': [{'c': 1}, {'b': 1, 'c': 1}]} mdict = DotDict(res) expect = 1 result = mdict.get('a.b') self.assertEqual(expect, result)
def test_DotDict_values(self): """Test DotDict get_values method""" res = {'a': [{'b': 1, 'c': 1}, {'c': 2}]} mdict = DotDict(res) expect = [1] result = [r for r in mdict.get_values('a.b')] self.assertEqual(expect, result) expect = [1, 2] result = [r for r in mdict.get_values('a.c')] self.assertEqual(expect, result) res = {'a': [{'b': [{'c': 2}, {'c': 3}]}, {'b': [{'c': 4}, {'c': 5}]}]} mdict = DotDict(res) expect = [2, 3, 4, 5] result = [r for r in mdict.get_values('a.b.c')] self.assertEqual(expect, result)
def test_set(self): """test set method""" rec = DotDict(self.rec1) rec['a.d'] = 111 self.assertEqual(rec['a.d'], 111) self.assertRaises(Exception, rec.set, ('a.c.d', 1))
def test_get_values(self): """test get_values method""" rec = DotDict(self.rec2) expect = [1, 2, 10, 20] result = [o for o in rec.get_values('a.c')] self.assertEqual(expect, result)
class UfoJob(UfoJob, LogMixin): def __init__(self, profiling=False, schedfixed=False, deviceCPU=False): self._profiling = profiling self._schedfixed = schedfixed self._deviceCPU = deviceCPU self.pm = PluginManager() self.graph = TaskGraph() self.tasks = DotDict() def init_scheduler(self): if self._deviceCPU: self.logger.debug('run on cpu') self.resource = Ufo.Resources(device_type=Ufo.DeviceType.CPU) else: self.logger.debug('run on gpu') self.resource = Ufo.Resources(device_type=Ufo.DeviceType.GPU) if self._schedfixed: self.logger.debug('fixed Scheduler') self.sched = Ufo.FixedScheduler() else: self.logger.debug('normal Scheduler') self.sched = Ufo.Scheduler() self.sched.set_resources(self.resource) self.sched.set_property('enable_tracing', self._profiling) def run_t(self, n=1): self.init_scheduler() def timer_function(): self.sched.run(self.graph) tm = Timer(timer_function) t = tm.timeit(n) # print self.sched # print self.resource return t @property def profiling(self): return self._profiling @profiling.setter def profiling(self, value): self._profiling = value @property def schedfixed(self): return self._schedfixed @schedfixed.setter def schedfixed(self, value): self._schedfixed = value @property def deviceCPU(self): return self._deviceCPU @deviceCPU.setter def deviceCPU(self, value): self._deviceCPU = value def branch(self, *args): try: b = [self.tasks.get(n) for n in args] self.graph.connect_branch(b) except KeyError: sys.exit("task %s does not exists" % n) return b def add_copy_task(self, taskname): task = Ufo.CopyTask() self.tasks[taskname] = task def add_task(self, taskname, plugin=None, **kargs): if self.tasks.get('taskname'): print 'Warning: %s already exists' % taskname if plugin is None: plugin = taskname task = self.pm.get_task(plugin, **kargs) self.tasks[taskname] = task
def test_delete(self): """test delete method""" rec = DotDict(self.rec1) rec.delete('a.c') expect = {'a': {'b': 1}} self.assertEqual(expect, rec)
def test_delete(self): """test delete method""" rec = DotDict(self.rec1) rec.delete("a.c") expect = {"a": {"b": 1}} self.assertEqual(expect, rec)
def test_get_values(self): """test get_values method""" rec = DotDict(self.rec2) expect = [1, 2, 10, 20] result = [o for o in rec.get_values("a.c")] self.assertEqual(expect, result)
def test_DotDict(self): """Test DotDict class""" res = {u'zip': {u'code': u'14850'}} mdict = DotDict(res) mdict['zip.code'] = 14850 expect = {u'zip': {u'code': 14850}} self.assertEqual(expect, mdict) res = {'a': {'b': {'c': 10}, 'd': 10}} mdict = DotDict(res) mdict['x.y.z'] = 10 expect = {'a': {'b': {'c': 10}, 'd': 10}, 'x': {'y': {'z': 10}}} self.assertEqual(expect, mdict) mdict['a.b.k.m'] = 10 expect = { 'a': { 'b': { 'c': 10, 'k': { 'm': 10 } }, 'd': 10 }, 'x': { 'y': { 'z': 10 } } } self.assertEqual(expect, mdict) expect = 10 result = mdict.get('a.b.k.m') self.assertEqual(expect, result) res = {'a': {'b': {'c': 10}, 'd': [{'x': 1}, {'x': 2}]}} mdict = DotDict(res) expect = 1 result = mdict.get('a.d.x') self.assertEqual(expect, result) expect = None result = mdict.get('a.M.Z') self.assertEqual(expect, result) res = {'a': {'b': {'c': 1, 'd': 2}}} mdict = DotDict(res) expect = {'a': {'b': {'c': 1}}} mdict.delete('a.b.d') self.assertEqual(expect, mdict)
class PivJob(UfoJob): def __init__(self, parms={}): super(PivJob, self).__init__(profiling=False, schedfixed=True, deviceCPU=False) self.parms = DotDict(default_parms) self.parms.update(parms) self.parms.ring_number = \ (self.parms.ring_end - self.parms.ring_start) / self.parms.ring_step + 1 self.setup_basic_tasks() def setup_basic_tasks(self): p = self.parms sc = self.parms.scale self.add_task('crop', x=p.xshift, y=p.yshift, width=1024, height=1024) self.add_task('contrast', 'piv_contrast') self.add_task('rescale', factor=1.0/sc) self.add_task('input_fft', 'fft', dimensions=2) self.add_task('ring_fft', 'fft', dimensions=2) self.add_task('ring_stack', 'stack', number=p.ring_number) self.add_task('ring_loop', 'loop', count=p.number) self.add_task('ring_convolution', 'complex_mult') self.add_task('ring_slice', 'slice') self.add_task('ring_pattern', start=p.ring_start/sc, end=p.ring_end/sc, step=p.ring_step/sc, thickness=p.ring_thickness/sc, method=p.ring_method, width=1024/sc, height=1024/sc) self.add_task('hessian_kernel', sigma=2.0/sc, width=1024/sc, height=1024/sc) self.add_task('hessian_fft', 'fft', dimensions=2) self.add_task('hessian_loop', 'loop', count=p.number*p.ring_number) self.add_task('hessian_convolution', 'complex_mult') self.add_task('hessian_analysis') self.add_copy_task('hessian_broadcast') self.add_task('ifft', dimensions=2) self.add_task('local_maxima', sigma=p.maxima_sigma) self.add_task('label_cluster', 'label-cluster') self.add_task('combine_test', 'combine-test') self.add_task('blob_test', alpha=p.blob_alpha) self.add_task('sum') self.add_task('ring_writer') p = self.parms scale = p.scale self.add_task('read', path=p.in_path, number=p.number, start=p.start) self.add_task('write', filename=p.out_file) self.add_task('stack1', 'stack', number=p.ring_number) self.add_task('stack2', 'stack', number=p.ring_number) self.add_task('stack3', 'stack', number=p.ring_number) self.add_task('unstack1', 'slice') self.add_task('unstack2', 'slice') self.add_task('m1', 'monitor') self.add_task('m2', 'monitor') self.add_task('m3', 'monitor') self.add_task('m4', 'monitor') self.add_task('m5', 'monitor') self.add_task('m6', 'monitor') self.add_task('m7', 'monitor') self.add_task('null') def setup_tasks(self): pass def setup_graph(self, flag): if flag == 1: b1 = self.branch('read', 'crop', 'rescale', 'contrast', 'input_fft')#, 'm1') b2 = self.branch('ring_pattern', 'ring_stack', 'ring_fft', 'ring_loop')#, 'm2') b3 = self.branch('ring_convolution', 'ifft', 'write') self.graph.merge_branch(b1, b2, b3) elif flag == 2: b1 = self.branch('read', 'crop', 'rescale', 'contrast', 'input_fft') b2 = self.branch('ring_pattern', 'ring_stack', 'ring_fft', 'ring_loop') b3 = self.branch('ring_convolution', 'ring_slice') self.graph.merge_branch(b1, b2, b3) b4 = self.branch('hessian_kernel', 'hessian_fft', 'hessian_loop') b5 = self.branch('hessian_convolution', 'ifft', 'hessian_analysis', 'write') self.graph.merge_branch(b3, b4, b5) else: b1 = self.branch('read', 'crop', 'rescale', 'contrast', 'input_fft') b2 = self.branch('ring_pattern', 'ring_stack', 'ring_fft', 'ring_loop') b3 = self.branch('ring_convolution', 'ring_slice') self.graph.merge_branch(b1, b2, b3) b4 = self.branch('hessian_kernel', 'hessian_fft', 'hessian_loop') b5 = self.branch('hessian_convolution', 'ifft', 'hessian_analysis', 'hessian_broadcast', 'stack1', 'unstack1') self.graph.merge_branch(b3, b4, b5) b6 = self.branch('hessian_broadcast', 'local_maxima', 'label_cluster', 'stack2', 'combine_test', 'unstack2') b7 = self.branch('blob_test', 'stack3', 'sum', 'write') self.graph.merge_branch(b5, b6, b7) def run(self, flag=None): self.setup_tasks() self.setup_graph(flag) self.log_tasks() runtime = self.run_t() self.logger.info('') self.logger.info('Program finished in %s seconds' % runtime ) def log_tasks(self): scale = self.tasks.rescale.props.factor fmt_f = "%16s.%-16s= %8.3f" fmt_i = "%16s.%-16s= %8i" self.logger.info( 'input: ' + self.tasks.read.props.path) self.logger.info( 'output: ' + self.tasks.write.props.filename) self.logger.info( '') self.logger.info( fmt_i % ('Ring', 'start', self.tasks.ring_pattern.props.start/scale)) self.logger.info( fmt_i % ('Ring', 'end', self.tasks.ring_pattern.props.end/scale)) self.logger.info( fmt_i % ('Ring', 'step', self.tasks.ring_pattern.props.step/scale)) self.logger.info( fmt_f % ('Contrast', 'low_cut', self.tasks.contrast.props.c1)) self.logger.info( fmt_f % ('Contrast', 'high_cut', self.tasks.contrast.props.c2)) self.logger.info( fmt_f % ('Contrast', 'shift', self.tasks.contrast.props.c3)) self.logger.info( fmt_f % ('Contrast', 'width', self.tasks.contrast.props.c4)) self.logger.info( fmt_f % ('Contrast', 'gamma', self.tasks.contrast.props.gamma)) self.logger.info( fmt_f % ('LocalMax', 'threshold', self.tasks.local_maxima.props.sigma)) self.logger.info( fmt_f % ('BlobTest', 'alpha', self.tasks.blob_test.props.alpha))