def test_res7(self):
rm = taskrun.ResourceManager(
taskrun.CounterResource('core', 9999, 4),
taskrun.CounterResource('mem', 9999, 8000))
ob = ComparisonCheckObserver(8, [
'+t1 < +t2 +t3 +t4 -t4 -t3 -t2 -t1',
'+t2 < +t3 +t4 -t4 -t3 -t2 -t1', '+t3 < +t4 -t4 -t3 -t2 -t1',
'+t4 < -t4 -t3 -t2 -t1'
],
verbose=False)
tm = taskrun.TaskManager(resource_manager=rm, observers=[ob])
t1 = taskrun.ProcessTask(tm, 't1', 'sleep 0.01')
t1.resources = {'core': 1, 'mem': 0}
t1.priority = 4
t2 = taskrun.ProcessTask(tm, 't2', 'sleep 0.02')
t2.resources = {'core': 1, 'mem': 0}
t2.priority = 3
t3 = taskrun.ProcessTask(tm, 't3', 'sleep 0.03')
t3.resources = {'core': 1, 'mem': 0}
t3.priority = 2
t4 = taskrun.ProcessTask(tm, 't4', 'sleep 0.04')
t4.resources = {'core': 1, 'mem': 0}
t4.priority = 1
tm.run_tasks()
self.assertTrue(ob.ok())
def test_blind_sequence(self):
ob = OrderCheckObserver(['@t1', '@t2', '+t1', '!t1', '+t2', '!t2'])
tm = taskrun.TaskManager(observers=[ob], failure_mode='blind_continue')
t1 = taskrun.ProcessTask(tm, 't1', 'false')
t2 = taskrun.ProcessTask(tm, 't2', 'false')
t2.add_dependency(t1)
tm.run_tasks()
self.assertTrue(ob.ok())
def test_dep3(self): ob = OrderCheckObserver(['@t1', '@t2', '+t2', '-t2', '+t1', '-t1']) tm = taskrun.TaskManager(observers=[ob]) t1 = taskrun.ProcessTask(tm, 't1', 'sleep 0.01') t2 = taskrun.ProcessTask(tm, 't2', 'sleep 0.01') t1.add_dependency(t2) tm.run_tasks() self.assertTrue(ob.ok())
def test_pri2(self):
ob = ComparisonCheckObserver(4, ['+t2 < -t2 +t1 -t1', '+t1 < -t1'],
verbose=False)
tm = taskrun.TaskManager(observers=[ob])
t1 = taskrun.ProcessTask(tm, 't1', 'sleep 0.01')
t2 = taskrun.ProcessTask(tm, 't2', 'sleep 0.02')
t1.priority = 1
t2.priority = 2
tm.run_tasks()
self.assertTrue(ob.ok())
def test_nocycle1(self): tm = taskrun.TaskManager() t1 = taskrun.ProcessTask(tm, 't1', '') t2 = taskrun.ProcessTask(tm, 't2', '') t3 = taskrun.ProcessTask(tm, 't3', '') t4 = taskrun.ProcessTask(tm, 't4', '') t5 = taskrun.ProcessTask(tm, 't5', '') t1.add_dependency(t2) t2.add_dependency(t3) t3.add_dependency(t4) t4.add_dependency(t5)
def test_pri5(self):
tm = taskrun.TaskManager(priority_levels=3)
t1 = taskrun.ProcessTask(tm, 't1', 'sleep 0.01')
t2 = taskrun.ProcessTask(tm, 't2', 'sleep 0.02')
t1.priority = 2
t2.priority = 1
success = True
try:
tm.run_tasks()
except AssertionError:
success = False
self.assertTrue(success)
def test_deepcyclic(self):
tm = taskrun.TaskManager()
t1 = taskrun.ProcessTask(tm, 't1', '')
t2 = taskrun.ProcessTask(tm, 't2', '')
t3 = taskrun.ProcessTask(tm, 't3', '')
t4 = taskrun.ProcessTask(tm, 't4', '')
t5 = taskrun.ProcessTask(tm, 't5', '')
t1.add_dependency(t2)
t2.add_dependency(t3)
t3.add_dependency(t4)
t4.add_dependency(t5)
with self.assertRaises(ValueError):
t5.add_dependency(t1)
def test_nooutputfiles(self):
fd, file1 = tempfile.mkstemp()
fd, file2 = tempfile.mkstemp()
fd, file3 = tempfile.mkstemp()
os.remove(file3)
with open(file1, 'w') as fd:
print('hello file1', file=fd)
with open(file2, 'w') as fd:
print('hello file2', file=fd)
time.sleep(0.01)
ob = taskrun.FileCleanupObserver()
tm = taskrun.TaskManager(observers=[ob])
t1 = taskrun.ProcessTask(tm, 't1', 'false'.format(file1, file2, file3))
t1.add_condition(
taskrun.FileModificationCondition([file1, file2], [file3]))
tm.run_tasks()
self.assertTrue(os.path.isfile(file1))
self.assertTrue(os.path.isfile(file2))
self.assertFalse(os.path.isfile(file3))
# remove all files
os.remove(file1)
os.remove(file2)
def test_aggressive_none(self):
ob = OrderCheckObserver(['@t1', '+t1', '-t1'])
tm = taskrun.TaskManager(observers=[ob],
failure_mode='aggressive_fail')
t1 = taskrun.ProcessTask(tm, 't1', 'true')
tm.run_tasks()
self.assertTrue(ob.ok())
def test_active_single(self):
ob = OrderCheckObserver(['@t1', '+t1', '!t1'])
tm = taskrun.TaskManager(observers=[ob],
failure_mode='active_continue')
t1 = taskrun.ProcessTask(tm, 't1', 'false')
tm.run_tasks()
self.assertTrue(ob.ok())
def test_res1(self):
rm = taskrun.ResourceManager(
taskrun.CounterResource('core', 9999, 4),
taskrun.CounterResource('mem', 9999, 8000))
tm = taskrun.TaskManager(resource_manager=rm)
t1 = taskrun.ProcessTask(tm, 't1', '')
with self.assertRaises(ValueError):
tm.run_tasks()
def test_mem1(self):
rm = taskrun.ResourceManager(taskrun.MemoryResource('ram', 9999, 1))
ob = OrderCheckObserver(['@t1', '+t1', '-t1'], verbose=False)
tm = taskrun.TaskManager(resource_manager=rm, observers=[ob])
t1 = taskrun.ProcessTask(tm, 't1', 'sleep 0.01')
t1.resources = {'ram': 1}
tm.run_tasks()
self.assertTrue(ob.ok())
def test_pri4(self):
ob = ComparisonCheckObserver(8, [
'+t2 < -t2 +t1 -t1 +t4 -t4 +t3 -t3', '+t1 < -t1 +t4 -t4 +t3 -t3',
'+t4 < -t4 +t3 -t3', '+t3 < -t3'
],
verbose=False)
tm = taskrun.TaskManager(observers=[ob])
t1 = taskrun.ProcessTask(tm, 't1', 'sleep 0.01')
t2 = taskrun.ProcessTask(tm, 't2', 'sleep 0.02')
t3 = taskrun.ProcessTask(tm, 't3', 'sleep 0.03')
t4 = taskrun.ProcessTask(tm, 't4', 'sleep 0.04')
t1.priority = 3
t2.priority = 4
t3.priority = 1
t4.priority = 2
tm.run_tasks()
self.assertTrue(ob.ok())
def test_passive_multiroottree_wide(self):
ob = ComparisonCheckObserver(8, [
'+t1 < +t2 +t3 +t4 !t3 -t4 -t2 -t1', '+t2 > +t1',
'+t2 < +t3 +t4 !t3 -t4 -t2 -t1', '+t3 > +t2 +t1',
'+t3 < +t4 !t3 -t4 -t2 -t1', '+t4 > +t3 +t2 +t1',
'+t4 < !t3 -t4 -t2 -t1', '!t3 > +t4 +t3 +t2 +t1',
'!t3 < -t4 -t2 -t1', '-t4 > !t3 +t4 +t3 +t2 +t1',
'-t2 > !t3 +t4 +t3 +t2 +t1', '-t1 > !t3 +t4 +t3 +t2 +t1'
],
verbose=False)
rm = taskrun.ResourceManager(taskrun.CounterResource('slot', 1, 1000))
tm = taskrun.TaskManager(observers=[ob],
resource_manager=rm,
failure_mode='passive_fail')
t1 = taskrun.ProcessTask(tm, 't1', 'sleep 0.04')
t1.priority = 4
t2 = taskrun.ProcessTask(tm, 't2', 'sleep 0.03')
t2.priority = 3
t3 = taskrun.ProcessTask(tm, 't3', 'sleep 0.01; false')
t3.priority = 2
t4 = taskrun.ProcessTask(tm, 't4', 'sleep 0.02')
t4.priority = 1
t3a = taskrun.ProcessTask(tm, 't3a', '')
t3a.add_dependency(t3)
t3b = taskrun.ProcessTask(tm, 't3b', '')
t3b.add_dependency(t3)
t3ba = taskrun.ProcessTask(tm, 't3ba', '')
t3ba.add_dependency(t3b)
tm.run_tasks()
self.assertTrue(ob.ok())
def test_active_multiroottree_narrow(self):
ob = ComparisonCheckObserver(8, [
'+t1 < -t1 +t2 -t2 +t3 !t3 +t4 -t4', '-t1 > +t1',
'-t1 < +t2 -t2 +t3 !t3 +t4 -t4', '+t2 > -t1 +t1',
'+t2 < -t2 +t3 !t3 +t4 -t4', '-t2 > +t2 -t1 +t1',
'-t2 < +t3 !t3 +t4 -t4', '+t3 > -t2 +t2 -t1 +t1',
'+t3 < !t3 +t4 -t4', '!t3 > +t3 -t2 +t2 -t1 +t1', '!t3 < +t4 -t4',
'+t4 > !t3 +t3 -t2 +t2 -t1 +t1', '+t4 < -t4',
'-t4 > +t4 !t3 +t3 -t2 +t2 -t1 +t1'
],
verbose=False)
rm = taskrun.ResourceManager(taskrun.CounterResource('slot', 1, 1))
tm = taskrun.TaskManager(observers=[ob],
resource_manager=rm,
failure_mode='active_continue')
t1 = taskrun.ProcessTask(tm, 't1', '')
t1.priority = 4
t2 = taskrun.ProcessTask(tm, 't2', '')
t2.priority = 3
t3 = taskrun.ProcessTask(tm, 't3', 'false')
t3.priority = 2
t4 = taskrun.ProcessTask(tm, 't4', '')
t4.priority = 1
t3a = taskrun.ProcessTask(tm, 't3a', '')
t3a.add_dependency(t3)
t3b = taskrun.ProcessTask(tm, 't3b', '')
t3b.add_dependency(t3)
t3ba = taskrun.ProcessTask(tm, 't3ba', '')
t3ba.add_dependency(t3b)
tm.run_tasks()
self.assertTrue(ob.ok())
def test_proc1(self):
fd, filename = tempfile.mkstemp()
tm = taskrun.TaskManager()
t1 = taskrun.ProcessTask(tm, 't1', 'rm {0}'.format(filename))
with open(filename, 'w') as fd:
print('hello world!\n', file=fd)
self.assertTrue(os.path.isfile(filename))
tm.run_tasks()
self.assertFalse(os.path.isfile(filename))
def set_task_function(tm, name, cmd, console_out, task_type, config):
task = taskrun.ProcessTask(tm, name, cmd)
if console_out:
task.stdout_file = console_out
task.stderr_file = console_out
if task_type is 'sim':
task.resources = {'cpus': 1, 'mem': 5}
else:
task.resources = {'cpus': 1, 'mem': 3}
return task
def test_cputime2(self):
rm = taskrun.ResourceManager(taskrun.CpuTimeResource('time', 10))
ob = OrderCheckObserver(['@t1', '+t1', '-t1'], verbose=False)
tm = taskrun.TaskManager(resource_manager=rm, observers=[ob])
t1 = taskrun.ProcessTask(tm, 't1', 'test/testprogs/burncycles 1 0.1')
t1.resources = {'time': 10}
tm.run_tasks()
self.assertTrue(t1.stdout.find('count=10') >= 0)
self.assertTrue(t1.stdout.find('completed') >= 0)
self.assertTrue(ob.ok())
def test_res2(self):
rm = taskrun.ResourceManager(
taskrun.CounterResource('core', 9999, 4),
taskrun.CounterResource('mem', 9999, 8000))
ob = OrderCheckObserver(['@t1', '+t1', '-t1'])
tm = taskrun.TaskManager(resource_manager=rm, observers=[ob])
t1 = taskrun.ProcessTask(tm, 't1', '')
t1.resources = {'core': 1, 'mem': 5000}
tm.run_tasks()
self.assertTrue(ob.ok())
def test_simple(self):
obs = [
OrderCheckObserver(['@t1', '+t1', '-t1']),
OrderCheckObserver(['@t1', '+t1', '-t1']),
OrderCheckObserver(['@t1', '+t1', '-t1'])
]
tm = taskrun.TaskManager(observers=obs)
t1 = taskrun.ProcessTask(tm, 't1', '')
tm.run_tasks()
for ob in obs:
self.assertTrue(ob.ok())
def test_mem5(self):
rm = taskrun.ResourceManager(taskrun.MemoryResource('ram', 9999, 0.25))
ob = OrderCheckObserver(['@t1', '+t1', '!t1'], verbose=False)
tm = taskrun.TaskManager(resource_manager=rm, observers=[ob])
t1 = taskrun.ProcessTask(tm, 't1',
'test/testprogs/alloclots 104857600 1000 5')
t1.resources = {'ram': 0.25}
tm.run_tasks()
self.assertTrue(t1.stdout.find('+blocks=2') >= 0)
self.assertTrue(t1.stdout.find('all allocated') < 0)
self.assertTrue(ob.ok())
def test_res8(self):
for div in [1, 2, 3, 10, 25, 50, 100, 187]:
cr = taskrun.CounterResource('foo', 4.3 / div, 4.3)
rm = taskrun.ResourceManager(cr)
tm = taskrun.TaskManager(resource_manager=rm)
rnd = random.Random(12345678)
for t in range(10000):
cmd = 'usleep {}'.format(rnd.gauss(1000, 500))
taskrun.ProcessTask(tm, str(t), cmd)
tm.run_tasks()
self.assertEqual(cr.total, 4.3)
self.assertEqual(cr.used, 0)
def create_task(tm, name, cmd, console_out, task_type, config):
task = taskrun.ProcessTask(tm, name, cmd)
if console_out:
task.stdout_file = console_out
task.stderr_file = console_out
if task_type == 'sim':
task.resources = {'cpus': 1, 'mem': 0.5}
elif task_type == 'parse':
task.resources = {'cpus': 1, 'mem': 0.5}
else:
task.resources = {'cpus': 1, 'mem': 1.1}
return task
def test_blind_4wide(self):
ob = ComparisonCheckObserver(8, [
'+t1 < +t2 +t3 +t4 !t3 -t4 -t2 -t1', '+t2 > +t1',
'+t2 < +t3 +t4 !t3 -t4 -t2 -t1', '+t3 > +t2 +t1',
'+t3 < +t4 !t3 -t4 -t2 -t1', '+t4 > +t3 +t2 +t1',
'+t4 < !t3 -t4 -t2 -t1', '!t3 > +t4 +t3 +t2 +t1',
'!t3 < -t4 -t2 -t1', '-t4 > !t3 +t4 +t3 +t2 +t1',
'-t2 > !t3 +t4 +t3 +t2 +t1', '-t1 > !t3 +t4 +t3 +t2 +t1'
],
verbose=False)
tm = taskrun.TaskManager(observers=[ob], failure_mode='blind_continue')
t1 = taskrun.ProcessTask(tm, 't1', 'sleep 0.04')
t1.priority = 4
t2 = taskrun.ProcessTask(tm, 't2', 'sleep 0.03')
t2.priority = 3
t3 = taskrun.ProcessTask(tm, 't3', 'sleep 0.01; false')
t3.priority = 2
t4 = taskrun.ProcessTask(tm, 't4', 'sleep 0.02')
t4.priority = 1
tm.run_tasks()
self.assertTrue(ob.ok())
def test_res4(self):
rm = taskrun.ResourceManager(
taskrun.CounterResource('core', 9999, 4),
taskrun.CounterResource('mem', 9999, 8000))
ob = OrderCheckObserver([
'@t1', '@t2', '@t3', '@t4', '+t1', '-t1', '+t2', '-t2', '+t3',
'-t3', '+t4', '-t4'
])
tm = taskrun.TaskManager(resource_manager=rm, observers=[ob])
t1 = taskrun.ProcessTask(tm, 't1', '')
t1.resources = {'core': 1, 'mem': 0}
t2 = taskrun.ProcessTask(tm, 't2', '')
t2.resources = {'core': 1, 'mem': 0}
t3 = taskrun.ProcessTask(tm, 't3', '')
t3.resources = {'core': 1, 'mem': 0}
t4 = taskrun.ProcessTask(tm, 't4', '')
t4.resources = {'core': 1, 'mem': 0}
t4.add_dependency(t3)
t3.add_dependency(t2)
t2.add_dependency(t1)
tm.run_tasks()
self.assertTrue(ob.ok())
def test_res5(self):
rm = taskrun.ResourceManager(
taskrun.CounterResource('core', 9999, 1),
taskrun.CounterResource('mem', 9999, 8000))
ob = OrderCheckObserver([
'@t1', '@t2', '@t3', '@t4', '+t1', '-t1', '+t2', '-t2', '+t3',
'-t3', '+t4', '-t4'
])
tm = taskrun.TaskManager(resource_manager=rm, observers=[ob])
t1 = taskrun.ProcessTask(tm, 't1', '')
t1.resources = {'core': 1, 'mem': 0}
t1.priority = 4
t2 = taskrun.ProcessTask(tm, 't2', '')
t2.resources = {'core': 1, 'mem': 0}
t2.priority = 3
t3 = taskrun.ProcessTask(tm, 't3', '')
t3.resources = {'core': 1, 'mem': 0}
t3.priority = 2
t4 = taskrun.ProcessTask(tm, 't4', '')
t4.resources = {'core': 1, 'mem': 0}
t4.priority = 1
tm.run_tasks()
self.assertTrue(ob.ok())
def test_noinputfiles(self):
fd, file1 = tempfile.mkstemp()
fd, file2 = tempfile.mkstemp()
fd, file3 = tempfile.mkstemp()
os.remove(file1)
os.remove(file2)
os.remove(file3)
time.sleep(0.01)
ob = taskrun.FileCleanupObserver()
tm = taskrun.TaskManager(observers=[ob])
t1 = taskrun.ProcessTask(tm, 't1', 'false'.format(file1, file2, file3))
t1.add_condition(
taskrun.FileModificationCondition([file1, file2], [file3]))
tm.run_tasks()
self.assertFalse(os.path.isfile(file1))
self.assertFalse(os.path.isfile(file2))
self.assertFalse(os.path.isfile(file3))
def main(args):
# ensure rundir exists, if not make it
if not os.path.isdir(args.rundir):
try:
os.mkdir(args.rundir)
except:
error('couldn\'t create {0}'.format(args.rundir))
# ensure the supersim environment exists
if not os.path.isdir(args.ssenv):
error('{0} does not exist'.format(args.ssenv))
# create a task manager to handle all tasks
rm = taskrun.ResourceManager(
taskrun.CounterResource('cpus', 9999, args.cpus),
taskrun.MemoryResource('mem', 9999, args.mem))
ob = taskrun.VerboseObserver(description=args.verbose)
tm = taskrun.TaskManager(resource_manager=rm,
observer=ob,
failure_mode=taskrun.FailureMode.ACTIVE_CONTINUE)
# sweep params
sweepStart = 1
sweepStop = 100
sweepStep = 9
loads = [
'{0:.02f}'.format(x / 100)
for x in range(sweepStart, sweepStop + 1, sweepStep)
]
# sweep load
parse_tasks = []
for load in loads:
runId = '{0:02d}_{1}'.format(args.queue, load)
# create file names
supersim_bin = os.path.join(args.ssenv, 'poem_supersim', 'bin',
'supersim')
sslatency_bin = os.path.join(args.ssenv, 'sslatency', 'bin',
'sslatency')
settings_json = os.path.join(args.ssenv, 'poem_supersim', 'json',
'hierarchyhyperx_iq_GlueFactory1.json')
simout_log = os.path.join(args.rundir, 'simout_{0}.log'.format(runId))
messages_mpf = os.path.join(args.rundir,
'messages_{0}.mpf'.format(runId))
rates_csv = os.path.join(args.rundir, 'rates_{0}.csv'.format(runId))
channels_csv = os.path.join(args.rundir,
'channels_{0}.mpf'.format(runId))
messages_csv = os.path.join(args.rundir,
'messages_{0}.csv'.format(runId))
packets_csv = os.path.join(args.rundir,
'packets_{0}.csv'.format(runId))
aggregate_csv = os.path.join(args.rundir,
'aggregate_{0}.csv'.format(runId))
packets_png = os.path.join(args.rundir,
'packets_{0}.png'.format(runId))
# create simulation task
sim_cmd = ('{0} {1} '
'application.max_injection_rate=float={2} '
'application.message_log.file=string={3} '
'application.rate_log.file=string={4} '
'network.channel_log.file=string={5} '
'network.interface.init_credits=uint={6} '
'network.router.input_queue_depth=uint={6} ').format(
supersim_bin, settings_json, load, messages_mpf,
rates_csv, channels_csv, args.queue)
sim_task = taskrun.ProcessTask(tm, 'sim_{0}'.format(runId), sim_cmd)
sim_task.stdout_file = simout_log
sim_task.stderr_file = simout_log
sim_task.resources = {'cpus': 1, 'mem': 10}
sim_task.add_condition(
taskrun.FileModificationCondition(
[settings_json],
[simout_log, messages_mpf, rates_csv, channels_csv]))
# create parser task
parse_cmd = '{0} -m {1} -p {2} -a {3} {4}'.format(
sslatency_bin, messages_csv, packets_csv, aggregate_csv,
messages_mpf)
parse_task = taskrun.ProcessTask(tm, 'parse_{0}'.format(runId),
parse_cmd)
parse_task.resources = {'cpus': 1, 'mem': 2}
parse_task.add_dependency(sim_task)
parse_task.add_condition(
taskrun.FileModificationCondition(
[messages_mpf], [messages_csv, packets_csv, aggregate_csv]))
parse_tasks.append(parse_task)
# create plot task
plot_cmd = 'sslqp {0} {1}'.format(packets_csv, packets_png)
plot_task = taskrun.ProcessTask(tm, 'plot_{0}'.format(runId), plot_cmd)
plot_task.resources = {'cpus': 1, 'mem': 2}
plot_task.add_dependency(parse_task)
plot_task.add_condition(
taskrun.FileModificationCondition([packets_csv], [packets_png]))
# create a task to make a load latency graph
loadlat_file = os.path.join(args.rundir,
'load_latency_{0:02d}.png'.format(args.queue))
loadlat_cmd = ('ssllp --row Packet --title "QueueSize={0}" '
'{1} {2} {3} {4}').format(args.queue, loadlat_file,
sweepStart, sweepStop + 1,
sweepStep)
agg_files = []
for load in loads:
runId = '{0:02d}_{1}'.format(args.queue, load)
aggregate_csv = os.path.join(args.rundir,
'aggregate_{0}.csv'.format(runId))
loadlat_cmd += ' {0}'.format(aggregate_csv)
agg_files.append(aggregate_csv)
loadlat_task = taskrun.ProcessTask(tm, 'loadlat_{0}'.format(args.queue),
loadlat_cmd)
loadlat_task.resources = {'cpus': 1, 'mem': 2}
for dep in parse_tasks:
loadlat_task.add_dependency(dep)
loadlat_task.add_condition(
taskrun.FileModificationCondition(agg_files, [loadlat_file]))
# run all tasks
tm.run_tasks()
def test_many(self):
# create files for processes
procs = 100
files = []
for proc_id in range(procs):
fd, file_a = tempfile.mkstemp()
fd, file_b = tempfile.mkstemp()
fd, file_c = tempfile.mkstemp()
files.append([file_a, file_b, file_c])
# write something into A and B files
time.sleep(0.01)
for proc_id in range(procs):
with open(files[proc_id][0], 'w') as afile:
print('hello file A from proc {0}'.format(proc_id), file=afile)
with open(files[proc_id][1], 'w') as bfile:
print('hello file B from proc {0}'.format(proc_id), file=bfile)
# initial
ob = OccurredCheckObserver([], verbose=False)
tm = taskrun.TaskManager(observers=[ob])
evts = []
for proc_id in range(procs):
task = taskrun.ProcessTask(
tm, 't{0:04}'.format(proc_id),
'cat {0} {1} > {2}'.format(files[proc_id][0],
files[proc_id][1],
files[proc_id][2]))
cond = taskrun.FileModificationCondition(
[files[proc_id][0], files[proc_id][1]], [files[proc_id][2]])
task.add_condition(cond)
evts.append('@{0}'.format(task.name))
evts.append('+{0}'.format(task.name))
evts.append('-{0}'.format(task.name))
ob.reinit(evts)
tm.run_tasks()
self.assertTrue(ob.ok())
# no change
ob = OccurredCheckObserver([], verbose=False)
tm = taskrun.TaskManager(observers=[ob])
evts = []
for proc_id in range(procs):
task = taskrun.ProcessTask(
tm, 't{0:04}'.format(proc_id),
'cat {0} {1} > {2}'.format(files[proc_id][0],
files[proc_id][1],
files[proc_id][2]))
cond = taskrun.FileModificationCondition(
[files[proc_id][0], files[proc_id][1]], [files[proc_id][2]])
task.add_condition(cond)
evts.append('@{0}'.format(task.name))
evts.append('*{0}'.format(task.name))
ob.reinit(evts)
tm.run_tasks()
self.assertTrue(ob.ok())
# change some
rnd = random.Random()
ob = OccurredCheckObserver([], verbose=False)
tm = taskrun.TaskManager(observers=[ob])
evts = []
for proc_id in range(procs):
task = taskrun.ProcessTask(
tm, 't{0:04}'.format(proc_id),
'cat {0} {1} > {2}'.format(files[proc_id][0],
files[proc_id][1],
files[proc_id][2]))
cond = taskrun.FileModificationCondition(
[files[proc_id][0], files[proc_id][1]], [files[proc_id][2]])
task.add_condition(cond)
# randomly sabotage this task
evts.append('@{0}'.format(task.name))
if bool(rnd.getrandbits(1)):
evts.append('+{0}'.format(task.name))
evts.append('-{0}'.format(task.name))
# randomly select between sabotaging the input or output
if bool(rnd.getrandbits(1)):
# sabotage input (change input file)
with open(files[proc_id][rnd.getrandbits(1)],
'w+') as ifile:
print(str(rnd.getrandbits(8)), file=ifile)
else:
# sabotage output (delete output file)
os.remove(files[proc_id][2])
else:
evts.append('*{0}'.format(task.name))
ob.reinit(evts)
tm.run_tasks()
self.assertTrue(ob.ok())
# no change
ob = OccurredCheckObserver([], verbose=False)
tm = taskrun.TaskManager(observers=[ob])
evts = []
for proc_id in range(procs):
task = taskrun.ProcessTask(
tm, 't{0:04}'.format(proc_id),
'cat {0} {1} > {2}'.format(files[proc_id][0],
files[proc_id][1],
files[proc_id][2]))
cond = taskrun.FileModificationCondition(
[files[proc_id][0], files[proc_id][1]], [files[proc_id][2]])
task.add_condition(cond)
evts.append('@{0}'.format(task.name))
evts.append('*{0}'.format(task.name))
ob.reinit(evts)
tm.run_tasks()
self.assertTrue(ob.ok())
# change some
rnd = random.Random()
ob = OccurredCheckObserver([], verbose=False)
tm = taskrun.TaskManager(observers=[ob])
evts = []
for proc_id in range(procs):
task = taskrun.ProcessTask(
tm, 't{0:04}'.format(proc_id),
'cat {0} {1} > {2}'.format(files[proc_id][0],
files[proc_id][1],
files[proc_id][2]))
cond = taskrun.FileModificationCondition(
[files[proc_id][0], files[proc_id][1]], [files[proc_id][2]])
task.add_condition(cond)
# randomly sabotage this task
evts.append('@{0}'.format(task.name))
if bool(rnd.getrandbits(1)):
evts.append('+{0}'.format(task.name))
evts.append('-{0}'.format(task.name))
# randomly select between sabotaging the input or output
if bool(rnd.getrandbits(1)):
# sabotage input (change input file)
with open(files[proc_id][rnd.getrandbits(1)],
'w+') as ifile:
print(str(rnd.getrandbits(8)), file=ifile)
else:
# sabotage output (delete output file)
os.remove(files[proc_id][2])
else:
evts.append('*{0}'.format(task.name))
ob.reinit(evts)
tm.run_tasks()
self.assertTrue(ob.ok())
# remove all files
for proc_id in range(procs):
for tfile in files[proc_id]:
os.remove(tfile)
def test_simple(self):
fd, file1 = tempfile.mkstemp()
fd, file2 = tempfile.mkstemp()
fd, file3 = tempfile.mkstemp()
with open(file1, 'w') as fd:
print('hello file1', file=fd)
with open(file2, 'w') as fd:
print('hello file2', file=fd)
time.sleep(0.01)
# initial run
ob = OrderCheckObserver(['@t1', '+t1', '-t1'], verbose=False)
tm = taskrun.TaskManager(observers=[ob])
t1 = taskrun.ProcessTask(
tm, 't1', 'cat {0} {1} > {2}'.format(file1, file2, file3))
tm.run_tasks()
self.assertTrue(ob.ok())
# nothing changed
ob = OrderCheckObserver(['@t1', '*t1'], verbose=False)
tm = taskrun.TaskManager(observers=[ob])
t1 = taskrun.ProcessTask(
tm, 't1', 'cat {0} {1} > {2}'.format(file1, file2, file3))
c1 = taskrun.FileModificationCondition([file1, file2], [file3])
t1.add_condition(c1)
tm.run_tasks()
self.assertTrue(ob.ok())
# nothing changed
ob = OrderCheckObserver(['@t1', '*t1'], verbose=False)
tm = taskrun.TaskManager(observers=[ob])
t1 = taskrun.ProcessTask(
tm, 't1', 'cat {0} {1} > {2}'.format(file1, file2, file3))
c1 = taskrun.FileModificationCondition([file1, file2], [file3])
t1.add_condition(c1)
tm.run_tasks()
self.assertTrue(ob.ok())
# missing output
os.remove(file3)
ob = OrderCheckObserver(['@t1', '+t1', '-t1'], verbose=False)
tm = taskrun.TaskManager(observers=[ob])
t1 = taskrun.ProcessTask(
tm, 't1', 'cat {0} {1} > {2}'.format(file1, file2, file3))
c1 = taskrun.FileModificationCondition([file1, file2], [file3])
t1.add_condition(c1)
tm.run_tasks()
self.assertTrue(ob.ok())
# nothing changed
ob = OrderCheckObserver(['@t1', '*t1'], verbose=False)
tm = taskrun.TaskManager(observers=[ob])
t1 = taskrun.ProcessTask(
tm, 't1', 'cat {0} {1} > {2}'.format(file1, file2, file3))
c1 = taskrun.FileModificationCondition([file1, file2], [file3])
t1.add_condition(c1)
tm.run_tasks()
self.assertTrue(ob.ok())
# input changed
time.sleep(0.01)
with open(file1, 'w') as fd:
print('hello file1!', file=fd)
time.sleep(0.01)
ob = OrderCheckObserver(['@t1', '+t1', '-t1'], verbose=False)
tm = taskrun.TaskManager(observers=[ob])
t1 = taskrun.ProcessTask(
tm, 't1', 'cat {0} {1} > {2}'.format(file1, file2, file3))
c1 = taskrun.FileModificationCondition([file1, file2], [file3])
t1.add_condition(c1)
tm.run_tasks()
self.assertTrue(ob.ok())
# nothing changed
ob = OrderCheckObserver(['@t1', '*t1'], verbose=False)
tm = taskrun.TaskManager(observers=[ob])
t1 = taskrun.ProcessTask(
tm, 't1', 'cat {0} {1} > {2}'.format(file1, file2, file3))
c1 = taskrun.FileModificationCondition([file1, file2], [file3])
t1.add_condition(c1)
tm.run_tasks()
self.assertTrue(ob.ok())
# input changed (test add_input)
time.sleep(0.01)
with open(file1, 'w') as fd:
print('hello file1!', file=fd)
time.sleep(0.01)
ob = OrderCheckObserver(['@t1', '+t1', '-t1'], verbose=False)
tm = taskrun.TaskManager(observers=[ob])
t1 = taskrun.ProcessTask(
tm, 't1', 'cat {0} {1} > {2}'.format(file1, file2, file3))
c1 = taskrun.FileModificationCondition([file2], [file3])
c1.add_input(file1)
t1.add_condition(c1)
tm.run_tasks()
self.assertTrue(ob.ok())
# missing output (test add_output)
os.remove(file3)
ob = OrderCheckObserver(['@t1', '+t1', '-t1'], verbose=False)
tm = taskrun.TaskManager(observers=[ob])
t1 = taskrun.ProcessTask(
tm, 't1', 'cat {0} {1} > {2}'.format(file1, file2, file3))
c1 = taskrun.FileModificationCondition([file1, file2], [])
c1.add_output(file3)
t1.add_condition(c1)
tm.run_tasks()
self.assertTrue(ob.ok())
# nothing changed
ob = OrderCheckObserver(['@t1', '*t1'], verbose=False)
tm = taskrun.TaskManager(observers=[ob])
t1 = taskrun.ProcessTask(
tm, 't1', 'cat {0} {1} > {2}'.format(file1, file2, file3))
c1 = taskrun.FileModificationCondition([file1, file2], [file3])
t1.add_condition(c1)
tm.run_tasks()
self.assertTrue(ob.ok())
# remove all files
os.remove(file1)
os.remove(file2)
os.remove(file3)
