def execute(self):
self.cont_out.v1 = self.cont_in.v1 + 1.0
self.cont_out.v2 = self.cont_in.v2 + 1.0
self.cont_out.vt2.x = self.cont_in.vt2.x + 1.0
self.cont_out.vt2.y = self.cont_in.vt2.y + 1.0
self.cont_out.vt2.vt3.a = self.cont_in.vt2.vt3.a
self.cont_out.vt2.vt3.b = self.cont_in.vt2.vt3.b
if self.cont_in.data is not None:
with self.cont_in.data.open() as inp:
filename = '%s.data.vt' % self.name
with open(filename, 'w') as out:
out.write(inp.read())
self.cont_out.data = FileRef(filename, self)
if self.cont_in.vt2.data is not None:
with self.cont_in.vt2.data.open() as inp:
filename = '%s.data.vt2' % self.name
with open(filename, 'w') as out:
out.write(inp.read())
self.cont_out.vt2.data = FileRef(filename, self)
if self.cont_in.vt2.vt3.data is not None:
with self.cont_in.vt2.vt3.data.open() as inp:
filename = '%s.data.vt3' % self.name
with open(filename, 'w') as out:
out.write(inp.read())
self.cont_out.vt2.vt3.data = FileRef(filename, self)
def setUp(self):
# SimulationRoot is static and so some junk can be left
# over from other tests when running under nose, so
# set it to cwd here just to be safe
SimulationRoot.chroot(os.getcwd())
self.asm = set_as_top(Assembly())
obj = self.asm.add('scomp1', SimpleComp())
self.asm.add('scomp2', SimpleComp())
self.asm.driver.workflow.add(['scomp1', 'scomp2'])
with self.asm.dir_context:
filename = 'top.data.vt'
with open(filename, 'w') as out:
out.write('vt data\n')
obj.cont_in.data = FileRef(filename, self.asm)
filename = 'top.data.vt2'
with open(filename, 'w') as out:
out.write('vt2 data\n')
obj.cont_in.vt2.data = FileRef(filename, self.asm)
filename = 'top.data.vt3'
with open(filename, 'w') as out:
out.write('vt3 data\n')
obj.cont_in.vt2.vt3.data = FileRef(filename, self.asm)
class Passthrough(Component):
""" Copies input files (implicitly via local_path) to output. """
text_in = File(iotype='in', local_path='tout', legal_types=['xyzzy', 'txt'])
binary_in = File(iotype='in', local_path='bout')
text_out = File(FileRef('tout'), iotype='out')
binary_out = File(FileRef('bout', binary=True), iotype='out')
def execute(self):
""" File copies are performed implicitly. """
# We have to manually propagate 'extra_stuff' because the output
# FileRef isn't copied from the input FileRef.
self.binary_out.extra_stuff = self.binary_in.extra_stuff
class FComp(Component):
infile = File(iotype='in', local_path='input')
outfile = File(FileRef('output'), iotype='out')
outval = Int(iotype='out')
def execute(self):
""" Runs code and sets `outfile`. """
with open(self.infile.abspath()) as f:
s = f.read().strip()
val = int(s.split()[1])
self.outval = val + 1
fname = self.outfile.abspath()
print "about to open", fname
sys.stdout.flush()
with open(fname, 'w') as f:
print "writing %s %d %d to %s" % (self.name, self.outval,
MPI.COMM_WORLD.rank, fname)
sys.stdout.flush()
f.write("%s %d %d\n" %
(self.name, self.outval, MPI.COMM_WORLD.rank))
print "wrote %s %d %d to %s" % (self.name, self.outval,
MPI.COMM_WORLD.rank, fname)
sys.stdout.flush()
def execute(self):
""" Write test data to files. """
if self.write_files:
cwd = os.getcwd()
path = 'source.txt'
self._logger.debug("opening file '%s' in %s", path, cwd)
with open(path, 'w') as out:
out.write(self.text_data)
self.text_file = FileRef(path)
path = os.path.join('..', 'sub', 'source.bin')
self._logger.debug("opening file '%s' in %s", path, cwd)
with open(path, 'wb') as out:
cPickle.dump(self.sub.binary_data, out, 2)
self.sub.binary_file = FileRef(path, binary=True)
def test_unique(self):
logging.debug('')
logging.debug('test_unique')
model = Model()
## This part isn't valid any more because Assemblies now do not configure
## themselves unless they're part of a rooted hierarchy. That means in this
## case that model.a and model.b won't exist until set_as_top is called on model
#for comp in (model.a, model.b):
#self.assertEqual(comp.create_instance_dir, True)
#self.assertNotEqual(model.a.directory, 'a')
#self.assertNotEqual(model.b.directory, 'b')
set_as_top(model)
for comp in (model.a, model.b):
self.assertEqual(comp.create_instance_dir, False)
self.assertEqual(comp.return_code, 0)
self.assertEqual(comp.timed_out, False)
self.assertEqual(model.a.directory, 'a')
self.assertEqual(model.b.directory, 'b')
model.infile = FileRef(INP_FILE, model, input=True)
model.run()
for comp in (model.a, model.b):
self.assertEqual(comp.return_code, 0)
self.assertEqual(comp.timed_out, False)
with model.outfile.open() as inp:
result = inp.read()
self.assertEqual(result, INP_DATA)
def test_save_bad_filevar(self):
logging.debug('')
logging.debug('test_save_bad_filevar')
# Set file trait path outside model root.
self.model.Source.text_file = FileRef('/illegal')
code = 'self.model.save_to_egg(self.model.name, next_egg(), py_dir=PY_DIR)'
msg = "Egg_TestModel: Can't save, Egg_TestModel.Source.text_file path"
assert_raises(self, code, globals(), locals(), ValueError, msg)
def test_save_load(self):
logging.debug('')
logging.debug('test_save_load')
sleeper = set_as_top(Sleeper())
sleeper.name = 'Sleepy'
sleeper.infile = FileRef(INP_FILE, sleeper, input=True)
# Exercise check_save_load().
retcode = check_save_load(sleeper)
self.assertEqual(retcode, 0)
class Source(Component):
""" Produces files. """
write_files = Bool(True, iotype='in')
text_data = Str(iotype='in')
binary_data = List([1.0], iotype='in')
text_file = File(FileRef('source.txt', content_type='txt'), iotype='out')
binary_file = File(FileRef('source.bin', binary=True,
extra_stuff='Hello world!'), iotype='out')
def execute(self):
""" Write test data to files. """
if self.write_files:
out = open(self.text_file.path, 'w')
out.write(self.text_data)
out.close()
out = open(self.binary_file.path, 'wb')
cPickle.dump(list(self.binary_data), out, 2)
out.close()
def test_no_owner(self):
logging.debug('')
logging.debug('test_no_owner')
# No owner.
path = os.path.join(os.sep, 'xyzzy')
ref = FileRef(path)
msg = 'abspath() failed: no owner specified for FileRef'
assert_raises(self, 'ref.open()', globals(), locals(), ValueError, msg)
# Absolute FileRef.
path = os.path.join(os.sep, 'xyzzy')
ref = FileRef(path, self)
code = 'ref.open()'
msg = "Path '%s' is absolute and no path checker is available." % path
assert_raises(self, code, globals(), locals(), ValueError, msg)
# Relative FileRef.
path = 'xyzzy'
ref = FileRef(path, self)
msg = "Path '%s' is relative and no absolute directory is available." % path
assert_raises(self, code, globals(), locals(), ValueError, msg)
class Model(Assembly):
""" Run multiple `Unique` component instances. """
infile = File(iotype='in', local_path='input')
outfile = File(FileRef('output'), iotype='out')
def configure(self):
self.add('a', Unique())
self.add('b', Unique())
self.driver.workflow.add(['a', 'b'])
self.connect('infile', 'a.infile')
self.connect('a.outfile', 'b.infile')
self.connect('b.outfile', 'outfile')
def test_remote(self):
logging.debug('')
logging.debug('test_remote')
init_cluster(allow_shell=True)
sleeper = set_as_top(Sleeper())
sleeper.env_filename = ENV_FILE
sleeper.env_vars = {'SLEEP_DATA': 'Hello world!'}
sleeper.external_files.append(
FileMetadata(path=ENV_FILE, output=True))
sleeper.infile = FileRef(INP_FILE, sleeper, input=True)
sleeper.timeout = 5
sleeper.resources = {'min_cpus': 1}
sleeper.run()
self.assertEqual(sleeper.return_code, 0)
self.assertEqual(sleeper.timed_out, False)
self.assertEqual(os.path.exists(ENV_FILE), True)
with open(ENV_FILE, 'r') as inp:
data = inp.readline().rstrip()
self.assertEqual(data, sleeper.env_vars['SLEEP_DATA'])
with sleeper.outfile.open() as inp:
result = inp.read()
self.assertEqual(result, INP_DATA)
# Null input file.
sleeper.stdin = ''
assert_raises(self, 'sleeper.run()', globals(), locals(), ValueError,
": Remote execution requires stdin of DEV_NULL or"
" filename, got ''")
# Specified stdin, stdout, and join stderr.
with open('sleep.in', 'w') as out:
out.write('froboz is a pig!\n')
sleeper.stdin = 'sleep.in'
sleeper.stdout = 'sleep.out'
sleeper.stderr = ExternalCode.STDOUT
sleeper.run()
# Null stderr.
sleeper.stderr = None
sleeper.run()
class FComp2(Component):
inval = Int(iotype='in')
outfile = File(FileRef('output'), iotype='out')
outval = Int(iotype='out')
def execute(self):
self.outval = self.inval + 1
fname = self.outfile.abspath()
print "about to open", fname
sys.stdout.flush()
with open(fname, 'w') as f:
print "writing %s %d %d to %s" % (self.name, self.outval,
MPI.COMM_WORLD.rank, fname)
sys.stdout.flush()
f.write("%s %d %d\n" %
(self.name, self.outval, MPI.COMM_WORLD.rank))
print "wrote %s %d %d to %s" % (self.name, self.outval,
MPI.COMM_WORLD.rank, fname)
sys.stdout.flush()
def test_unique(self):
logging.debug('')
logging.debug('test_unique')
model = set_as_top(Model())
for comp in (model.a, model.b):
self.assertEqual(comp.create_instance_dir, False)
self.assertEqual(comp.return_code, 0)
self.assertEqual(comp.timed_out, False)
self.assertEqual(model.a.directory, 'a')
self.assertEqual(model.b.directory, 'b')
model.infile = FileRef(INP_FILE, model, input=True)
model.run()
for comp in (model.a, model.b):
self.assertEqual(comp.return_code, 0)
self.assertEqual(comp.timed_out, False)
with model.outfile.open() as inp:
result = inp.read()
self.assertEqual(result, INP_DATA)
class Sleeper(ExternalCode):
""" Used to test external code functionality. """
delay = Int(1, units='s', iotype='in')
env_filename = Str(iotype='in')
infile = File(iotype='in', local_path='input')
outfile = File(FileRef('output'), iotype='out')
def __init__(self):
super(Sleeper, self).__init__()
self.directory = DIRECTORY
self.external_files = [
FileMetadata(path='sleep.py', input=True, constant=True),
]
def execute(self):
""" Runs code and sets `outfile`. """
self.command = ['python', 'sleep.py', str(self.delay)]
if self.env_filename:
self.command.append(self.env_filename)
super(Sleeper, self).execute()
def get(self, obj, name):
"""
Get remote value and write to local file.
obj: object
Containing object, ignored.
name: string
Name in `obj`, ignored.
"""
if self._component._call_cpath_updated:
return None # Not initialized.
if self._client is None: # Happens during component.__setstate__
return None
binary = self._client.get(self._rpath+'.isBinary') == 'true'
valstr = self.rget()
mode = 'wb' if binary else 'w'
with self._component.dir_context:
with open(self._path, mode) as out:
out.write(valstr)
return FileRef(self._path, self._component, binary=binary)
desc='1D int array',
default_value=[1, 2, 3, 4, 5, 6, 7, 8, 9])
s1d = List(Str,
iotype='in',
desc='1D string array',
value=['Hello', 'from', 'TestComponent.SubGroup'])
flst = List(Float, iotype='in', desc='List of floats')
ilst = List(Int, iotype='in', desc='List of ints')
class Bogus(object):
""" To test instantiation. """
def __init__(self, need_one_argument):
self._arg = need_one_argument
if __name__ == '__main__':
top = Assembly()
comp = top.add('comp', TestComponent())
comp.in_file = FileRef('ASTestComp-0.1.cfg',
owner=top,
platform=sys.platform)
comp.run()
for path in ('x', 'y', 'z', 'exe_count', 'sub_group.b', 'sub_group.f',
'sub_group.i', 'sub_group.s', 'sub_group.fe', 'sub_group.ie',
'sub_group.se', 'sub_group.f1d', 'sub_group.i1d',
'sub_group.s1d'):
print '%s: %s' % (path, comp.get(path))
class TestComponent(Component, ASMixin):
""" Just something to test with. """
x = Float(iotype='in', default_value=2, desc='X input')
y = Float(iotype='in',
default_value=3,
desc='Y input',
low=-10,
high=10,
units='ft')
z = Float(iotype='out', desc='Z output', units='ft')
exe_count = Int(iotype='out', desc='Execution count')
exe_dir = Str(iotype='out', desc='Execution directory')
in_file = File(iotype='in', local_path='inFile.data', desc='Input file')
out_file = File(FileRef('outFile.data'), iotype='out', desc='Output file')
obj_input = VarTree(TopObj(iotype='in'))
obj_output = VarTree(TopObj(iotype='out'))
def __init__(self):
super(TestComponent, self).__init__()
self.add('sub_group', SubGroup())
def execute(self):
if self.x < 0:
raise RuntimeError('x %s is < 0' % self.x)
self.z = self.x * self.y
self.exe_count += 1
self.exe_dir = self.get_abs_directory()
with self.in_file.open() as inp:
with open(self.out_file.path, 'w') as out:
out.write(inp.read())
self._logger.info(' %s %s %s', self.x, self.y, self.z)
sys.stdout.write('stdout: %s %s %s\n' % (self.x, self.y, self.z))
sys.stdout.flush()
# sys.stderr.write('stderr: %s %s %s\n' % (self.x, self.y, self.z))
# sys.stderr.flush()
# Copy input object to output object.
self.obj_output.tob = self.obj_input.tob
self.obj_output.tof = self.obj_input.tof
self.obj_output.toi = self.obj_input.toi
self.obj_output.tos = self.obj_input.tos
self.obj_output.tofe = self.obj_input.tofe
self.obj_output.toie = self.obj_input.toie
self.obj_output.tose = self.obj_input.tose
self.obj_output.tof1d = self.obj_input.tof1d
self.obj_output.tof2d = self.obj_input.tof2d
self.obj_output.tof3d = self.obj_input.tof3d
self.obj_output.toi1d = self.obj_input.toi1d
self.obj_output.tos1d = self.obj_input.tos1d
self.obj_output.toflst = self.obj_input.toflst
self.obj_output.toilst = self.obj_input.toilst
self.obj_output.subobj.sob = self.obj_input.subobj.sob
self.obj_output.subobj.sof = self.obj_input.subobj.sof
self.obj_output.subobj.soi = self.obj_input.subobj.soi
self.obj_output.subobj.sos = self.obj_input.subobj.sos
@rbac(('owner', 'user'))
def cause_exception(self):
self.raise_exception("It's your own fault...", RuntimeError)
@rbac(('owner', 'user'))
def float_method(self):
return self.x + self.y
@rbac(('owner', 'user'))
def int_method(self):
return self.exe_count
@rbac(('owner', 'user'))
def null_method(self):
return
@rbac(('owner', 'user'))
def str_method(self):
msg = 'current state: x %r, y %r, z %r, exe_count %r' \
% (self.x, self.y, self.z, self.exe_count)
return msg
def test_normal(self):
logging.debug('')
logging.debug('test_normal')
sleeper = set_as_top(Sleeper())
sleeper.env_filename = ENV_FILE
sleeper.env_vars = {'SLEEP_DATA': 'Hello world!'}
sleeper.external_files.append(
FileMetadata(path=ENV_FILE, output=True))
sleeper.infile = FileRef(INP_FILE, sleeper, input=True)
sleeper.stderr = None
sleeper.run()
self.assertEqual(sleeper.return_code, 0)
self.assertEqual(sleeper.timed_out, False)
self.assertEqual(os.path.exists(ENV_FILE), True)
with open(ENV_FILE, 'rU') as inp:
data = inp.readline().rstrip()
self.assertEqual(data, sleeper.env_vars['SLEEP_DATA'])
with sleeper.outfile.open() as inp:
result = inp.read()
self.assertEqual(result, INP_DATA)
# Force an error.
sleeper.stderr = 'sleep.err'
sleeper.delay = -1
assert_raises(self, 'sleeper.run()', globals(), locals(), RuntimeError,
': return_code = 1')
sleeper.delay = 1
# Redirect stdout & stderr.
sleeper.env_vars = {'SLEEP_ECHO': '1'}
sleeper.stdin = 'sleep.in'
sleeper.stdout = 'sleep.out'
sleeper.stderr = 'sleep.err'
with open('sleep.in', 'w') as out:
out.write('Hello World!\n')
sleeper.run()
with open('sleep.out', 'r') as inp:
self.assertEqual(inp.read(), 'stdin echo to stdout\n'
'Hello World!\n')
with open('sleep.err', 'r') as inp:
self.assertEqual(inp.read(), 'stdin echo to stderr\n'
'Hello World!\n')
# Exercise check_files() errors.
os.remove('input')
assert_raises(self, 'sleeper.check_files(inputs=True)',
globals(), locals(), RuntimeError,
": missing 'in' file 'input'")
os.remove('output')
assert_raises(self, 'sleeper.check_files(inputs=False)',
globals(), locals(), RuntimeError,
": missing 'out' file 'output'")
os.remove('sleep.in')
assert_raises(self, 'sleeper.check_files(inputs=True)',
globals(), locals(), RuntimeError,
": missing stdin file 'sleep.in'")
os.remove('sleep.err')
assert_raises(self, 'sleeper.check_files(inputs=False)',
globals(), locals(), RuntimeError,
": missing stderr file 'sleep.err'")
os.remove('sleep.out')
assert_raises(self, 'sleeper.check_files(inputs=False)',
globals(), locals(), RuntimeError,
": missing stdout file 'sleep.out'")
# Show that non-existent expected files are detected.
sleeper.external_files.append(
FileMetadata(path='missing-input', input=True))
assert_raises(self, 'sleeper.run()',
globals(), locals(), RuntimeError,
": missing input file 'missing-input'")