def run_first(self, root):
print('run_first()')
db = StorageFilesystem(root, compress=True)
cc = Context(db=db)
#
cc.comp_dynamic(e1, job_id='e')
cc.batch_command('make recurse=1; ls')
def run_second(self, root):
print('run_second()')
db = StorageFilesystem(root, compress=True)
cc = Context(db=db)
#
cc.comp_dynamic(e2, job_id='e')
cc.batch_command('details e;clean;ls;make recurse=1')
def run_first(self, root):
db = StorageFilesystem(root, compress=True)
cc = Context(db=db)
#
cc.comp(g, job_id='g')
cc.comp(h, job_id='h')
cc.batch_command('make')
def run_second(self, root):
info('run_second()')
db = StorageFilesystem(root, compress=True)
cc = Context(db=db)
#
cc.comp_dynamic(f2, job_id='f')
cc.batch_command('clean;make recurse=1')
def main():
from compmake import Context
c = Context()
for param1 in [1, 2, 3]:
for param2 in [10, 11, 12]:
res1 = c.comp(func1, param1)
res2 = c.comp(func2, res1, param2)
c.comp(draw, res2)
# Run command passed on command line or otherwise run console.
cmds = sys.argv[1:]
if cmds:
c.batch_command(' '.join(cmds))
else:
print('Use "make recurse=1" or "parmake recurse=1" to make all.')
c.compmake_console()
def run_second(self, root):
print('run_second()')
db = StorageFilesystem(root, compress=True)
cc = Context(db=db)
#
cc.comp_dynamic(e2, job_id='e')
cc.batch_command('details e;clean;ls;make recurse=1')
def run_second(self, root):
info('run_second()')
db = StorageFilesystem(root, compress=True)
cc = Context(db=db)
#
cc.comp_dynamic(f2, job_id='f')
cc.batch_command('clean;make recurse=1')
def run_first(self, root):
db = StorageFilesystem(root, compress=True)
cc = Context(db=db)
#
cc.comp(g, job_id='g')
cc.comp(h, job_id='h')
cc.batch_command('make')
def run_first(self, root):
print('run_first()')
db = StorageFilesystem(root, compress=True)
cc = Context(db=db)
#
cc.comp_dynamic(e1, job_id='e')
cc.batch_command('make recurse=1; ls')
def main():
print('This is an example of how to use the "progress" function.')
from compmake import Context
c = Context()
c.comp(mylongfunction)
# Run command passed on command line or otherwise run console.
cmds = sys.argv[1:]
if cmds:
c.batch_command(' '.join(cmds))
else:
print('Use "make recurse=1" or "parmake recurse=1" to make all.')
c.compmake_console()
def main():
print('This is an example of how to use the "progress" function.')
from compmake import Context
c = Context()
c.comp(mylongfunction)
# Run command passed on command line or otherwise run console.
cmds = sys.argv[1:]
if cmds:
c.batch_command(' '.join(cmds))
else:
print('Use "make recurse=1" or "parmake recurse=1" to make all.')
c.compmake_console()
def test_dynamic6(self):
# first define with job and run
mockup6(self.cc, both=True)
db = self.db
self.assertRaises(CompmakeBug, jobs_defined, job_id='hd', db=db)
self.assert_cmd_success('make recurse=1')
check_job_cache_state(job_id='hd', states=[Cache.DONE], db=db)
self.assertEqual(jobs_defined(job_id='hd', db=db),
set(['hd-id']))
# self.assert_cmd_success('graph compact=0 color=0 '
# 'cluster=1 filter=dot')
self.assertJobsEqual('all', ['fd', 'fd-gd', 'fd-gd-g2',
'hd', 'hd-id', 'hd-id-i2',
'summary'])
self.assertJobsEqual('done', ['fd', 'fd-gd', 'fd-gd-g2',
'hd', 'hd-id', 'hd-id-i2',
'summary'])
# now redo it
self.db = StorageFilesystem(self.root, compress=True)
self.cc = Context(db=self.db)
print('running again with both=False')
mockup6(self.cc, both=False)
clean_other_jobs(context=self.cc)
self.assertJobsEqual('all', ['fd', 'fd-gd', 'fd-gd-g2',
'summary'])
job= get_job('summary', self.db)
print('job.children: %s' % job.children)
print('job.dynamic_children: %s' % job.dynamic_children)
self.assertEqual(job.dynamic_children, {'fd': set(['fd-gd'])})
self.assertEqualSet(direct_children('summary', self.db), ['fd', 'fd-gd'])
self.assert_cmd_success('ls')
self.assert_cmd_success('make recurse=1')
self.assertJobsEqual('all', ['fd', 'fd-gd', 'fd-gd-g2', 'summary'])
self.assertJobsEqual('done', ['fd', 'fd-gd', 'fd-gd-g2', 'summary'])
def funcB(res1, param_b):
print('funcB(%r, %r)' % (res1, param_b))
# we now add an exception
if param_b == 11:
msg = 'Exception raised for b = %d.' % param_b
raise Exception(msg)
return res1 + param_b
def draw(res2):
print('draw(%r)' % res2)
if __name__ == '__main__':
from compmake import Context
context = Context()
for param_a in [1, 2, 3]:
for param_b in [10, 11, 12]:
context.comp_prefix('a%s-b%s' % (param_a, param_b))
res1 = context.comp(funcA, param_a, job_id='preparing')
res2 = context.comp(funcB, res1, param_b, job_id='computing')
context.comp(draw, res2, job_id='drawing')
import sys
if len(sys.argv) == 1:
print('Presenting an interactive console')
context.compmake_console()
else:
print('Running the computation in batch mode')
cmd = " ".join(sys.argv[1:])
def f(x):
print('processing %s' % x)
if __name__ == '__main__':
from compmake import Context
c = Context()
for p in [42, 43, 44]:
c.comp(f, x=p)
c.batch_command('clean;parmake')
def f(x):
print('processing %s' % x)
if __name__ == '__main__':
from compmake import Context
c = Context()
for p in [42, 43, 44]:
c.comp(f, x=p)
c.batch_command('clean;parmake')
def funcB(res1, param_b):
print('funcB(%r, %r)' % (res1, param_b))
# we now add an exception
if param_b == 11:
msg = 'Exception raised for b = %d.' % param_b
raise Exception(msg)
return res1 + param_a
def draw(res2):
print('draw(%r)' % res2)
pass
if __name__ == '__main__':
from compmake import Context
context = Context()
for param_a in [1,2,3]:
for param_b in [10,11,12]:
context.comp_prefix('a%s-b%s' % (param_a, param_b))
res1 = context.comp(funcA, param_a, job_id='preparing')
res2 = context.comp(funcB, res1, param_b, job_id='computing')
context.comp(draw, res2, job_id='drawing')
import sys
if len(sys.argv) == 1:
print('Presenting an interactive console')
context.compmake_console()
else:
print('Running the computation in batch mode')
cmd = " ".join(sys.argv[1:])
def generate_tests(context, values):
res = []
for v in values:
res.append(context.comp(func1, v))
return context.comp(summary, res)
def summary(results):
print('I finished with this: %s' % results)
if __name__ == '__main__':
from compmake import Context
c = Context()
values = c.comp(cases)
# comp_dynamic gives the function an extra argument
# "context" to further define jobs
c.comp_dynamic(generate_tests, values)
# Run command passed on command line or otherwise run console.
import sys
cmds = sys.argv[1:]
if cmds:
c.batch_command(' '.join(cmds))
else:
print('Use "make recurse=1" (or "parmake") to make all.')
c.compmake_console()
def f(x):
print('f(x=%r)' % x)
return x * 2
def statistics(res):
print('statistics(res=%s)' % res.__repr__())
return sum(res)
def report(val):
print('The sum is: %r' % val)
if __name__ == '__main__':
from compmake import Context
c = Context()
params = [42, 43, 44]
jobs = [c.comp(f, x=p) for p in params]
summary = c.comp(statistics, jobs)
c.comp(report, summary)
c.batch_command('clean;parmake echo=1')
def f(x):
print('f(x=%r)' % x)
return x * 2
def schedule(context, params):
print('schedule(context, params=%s)' % params.__repr__())
for p in [42, 43, 44]:
context.comp(f, x=p)
if __name__ == '__main__':
from compmake import Context
c = Context()
c.comp_dynamic(schedule, params=[42, 43, 44])
c.batch_command('clean;parmake echo=1 recurse=1')
from mycomputations import funcA, funcB, draw
if __name__ == '__main__':
from compmake import Context
context = Context()
for param_a in [1, 2, 3]:
for param_b in [10, 11, 12]:
res1 = context.comp(funcA, param_a)
res2 = context.comp(funcB, res1, param_b)
context.comp(draw, res2)
context.compmake_console()
def f(x):
print('f(x=%r)' % x)
return x * 2
def statistics(res):
print('statistics(res=%s)' % res.__repr__())
return sum(res)
def report(val):
print('The sum is: %r' % val)
if __name__ == '__main__':
from compmake import Context
c = Context()
params = [42, 43, 44]
jobs = [c.comp(f, x=p) for p in params]
summary = c.comp(statistics, jobs)
c.comp(report, summary)
c.batch_command('clean;parmake echo=1')
def f(x):
print('f(x=%r)' % x)
return x * 2
def statistics(res):
print('statistics(res=%s)' % res.__repr__())
return sum(res)
def schedule(context, params):
print('schedule(context, params=%s)' % params.__repr__())
jobs = [context.comp(f, x=p) for p in params]
summary = context.comp(statistics, jobs)
# returns a job "promise", not a value!
return summary
def report(summary):
print('report(summary=%r)' % summary)
if __name__ == '__main__':
from compmake import Context
c = Context()
summary = c.comp_dynamic(schedule, [42, 43, 44])
c.comp(report, summary)
c.batch_command('clean;parmake echo=1 recurse=1')
return param1
def funcB(res1, param2):
print('funcB(%s, %s)' % (res1, param2))
time.sleep(1) # ... which takes some time
return res1 + param2
def draw(res2):
print('draw(%s)' % res2)
if __name__ == '__main__':
from compmake import Context
context = Context()
# A typical pattern: you want to try
# many combinations of parameters
for param1 in [1, 2, 3]:
for param2 in [10, 11, 12]:
# Simply use "y = comp(f, x)" whenever
# you would have used "y = f(x)".
res1 = context.comp(funcA, param1)
# You can use return values as well.
res2 = context.comp(funcB, res1, param2)
context.comp(draw, res2)
# Now, a few options to run this:
# 1) Call this file using the compmake program:
# $ python example.py
# and then run "make" at the prompt.
from filesystem import list_images, list_categories
from images_read_write import rgb_write
from instancing import reduce_list_as_tree
from process import average, process_one
import os
from process import process_cat
if __name__ == '__main__':
# Get the full path to the dataset
dataset = '101_ObjectCategories'
output = 'out'
from compmake import Context
context = Context()
cats = list_categories(dataset)
for category, path in cats.items():
out = os.path.join(output, '%s.jpg' % category)
context.comp(process_cat, path, out, job_id=category)
import sys
cmds = sys.argv[1:]
if cmds:
context.batch_command(" ".join(cmds))
else:
context.compmake_console()
def f(x):
print('f(x=%r)' % x)
return x * 2
def statistics(res):
print('statistics(res=%s)' % res.__repr__())
return sum(res)
def schedule(context, params):
print('schedule(context, params=%s)' % params.__repr__())
jobs = [context.comp(f, x=p) for p in params]
summary = context.comp(statistics, jobs)
# returns a job "promise", not a value!
return summary
def report(summary):
print('report(summary=%r)' % summary)
if __name__ == '__main__':
from compmake import Context
c = Context()
summary = c.comp_dynamic(schedule, [42, 43, 44])
c.comp(report, summary)
c.batch_command('clean;parmake echo=1 recurse=1')
from mycomputations import funcA, funcB, draw
if __name__ == '__main__':
from compmake import Context
context = Context()
for param_a in [1, 2, 3]:
for param_b in [10, 11, 12]:
# Add a prefix to the job ids for easy reference
prefix = 'a%s-b%s' % (param_a, param_b)
context.comp_prefix(prefix)
res1 = context.comp(funcA, param_a)
res2 = context.comp(funcB, res1, param_b)
context.comp(draw, res2)
context.compmake_console()
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from example_big_support import first, second, third, fail_randomly, failure_prob
if __name__ == '__main__':
from compmake import Context
c = Context()
branch = 10
print('We will now define a hierarchy of %d x %d x %d = %d jobs.' %
(branch, branch, branch, branch * branch * branch))
print('Each can fail randomly with probability %f.' % failure_prob)
# args = sys.argv[1:]
# if args:
# branch = int(args.pop(0))
for i in range(branch):
ijobs = []
for j in range(branch):
kjobs = []
for k in range(branch):
kjobs.append(c.comp(third, job_id='%d-%d-%d' % (i, j, k)))
ijobs.append(c.comp(second, kjobs, job_id='%d-%d' % (i, j)))
c.comp(first, ijobs, job_id='%d' % i)
# Run command passed on command line or otherwise run console.
import sys
cmds = sys.argv[1:]
if cmds:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
if __name__ == '__main__':
from compmake import Context
c = Context()
from example_external_support import generate_tests, cases
values = c.comp(cases)
# comp_dynamic gives the function an extra argument
# "context" to further define jobs
c.comp_dynamic(generate_tests, values)
# Run command passed on command line or otherwise run console.
import sys
cmds = sys.argv[1:]
if cmds:
c.batch_command(' '.join(cmds))
else:
print('Use "make recurse=1" or "parmake recurse=1" to make all.')
c.compmake_console()
def funcA(param1):
print('funcA(%s)' % param1)
time.sleep(1) # ... which takes some time
return param1
def funcB(res1, param2):
print('funcB(%s, %s)' % (res1, param2))
time.sleep(1) # ... which takes some time
return res1 + param2
def draw(res2):
print('draw(%s)' % res2)
if __name__ == '__main__':
from compmake import Context
context = Context()
# A typical pattern: you want to try
# many combinations of parameters
for param1 in [1,2,3]:
for param2 in [10,11,12]:
# Simply use "y = comp(f, x)" whenever
# you would have used "y = f(x)".
res1 = context.comp(funcA, param1)
# You can use return values as well.
res2 = context.comp(funcB, res1, param2)
context.comp(draw, res2)
# Now, a few options to run this:
# 1) Call this file using the compmake program:
# $ python example.py
# and then run "make" at the prompt.
def mylongfunction():
N = 4
for i in range(N):
progress('Task A', (i, N))
time.sleep(wait)
for i in range(N):
progress('Task B', (i, N))
time.sleep(wait)
if __name__ == '__main__':
print('This is an example of how to use the "progress" function.')
from compmake import Context
c = Context()
c.comp(mylongfunction)
# Run command passed on command line or otherwise run console.
cmds = sys.argv[1:]
if cmds:
c.batch_command(' '.join(cmds))
else:
print('Use "make recurse=1" or "parmake recurse=1" to make all.')
c.compmake_console()
from mycomputations import funcA, funcB, draw
if __name__ == '__main__':
from compmake import Context
context = Context()
for param_a in [1, 2, 3]:
for param_b in [10, 11, 12]:
prefix = 'a%s-b%s' % (param_a, param_b)
context.comp_prefix(prefix)
# use job_id to override default naming
res1 = context.comp(funcA, param_a, job_id='preparing')
res2 = context.comp(funcB, res1, param_b, job_id='computing')
context.comp(draw, res2, job_id='drawing')
context.compmake_console()
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from example_big_support import first,second,third, fail_randomly, failure_prob
if __name__ == '__main__':
from compmake import Context
c = Context()
branch = 10
print('We will now define a hierarchy of %d x %d x %d = %d jobs.'
% (branch, branch, branch, branch * branch * branch))
print('Each can fail randomly with probability %f.' % failure_prob)
# args = sys.argv[1:]
# if args:
# branch = int(args.pop(0))
for i in range(branch):
ijobs = []
for j in range(branch):
kjobs = []
for k in range(branch):
kjobs.append(c.comp(third, job_id='%d-%d-%d' % (i, j, k)))
ijobs.append(c.comp(second, kjobs, job_id='%d-%d' % (i, j)))
c.comp(first, ijobs, job_id='%d' % i)
# Run command passed on command line or otherwise run console.
import sys
cmds = sys.argv[1:]
if cmds:
def f(x):
print('f(x=%r)' % x)
return x * 2
def schedule(context, params):
print('schedule(context, params=%s)' % params.__repr__())
for p in [42, 43, 44]:
context.comp(f, x=p)
if __name__ == '__main__':
from compmake import Context
c = Context()
c.comp_dynamic(schedule, params=[42, 43, 44])
c.batch_command('clean;parmake echo=1 recurse=1')
from filesystem import list_categories
from instancing import instance_categories
import os
if __name__ == '__main__':
# Get the full path to the dataset
dataset = os.path.realpath('101_ObjectCategories')
output = os.path.realpath('out')
from compmake import Context
context = Context()
categories = context.comp(list_categories, dataset)
context.comp_dynamic(instance_categories, categories, output)
import sys
cmds = sys.argv[1:]
if cmds:
context.batch_command(" ".join(cmds))
else:
context.compmake_console()