def test_additional_bindings(self):
# Some additional stuff to bind into docker
tempDir = tempfile.mkdtemp()
tempFile = tempfile.mktemp()
with open(tempFile, 'w') as f:
f.write('data')
# One binding specifies the target path in the container, the other doesn't
# so it defaults to the same path
a = DockerApp(
'a',
'a',
image='ubuntu:14.04',
command="cp /opt/file %s" % (tempDir,),
additionalBindings=[tempDir, "%s:/opt/file" % (tempFile,)]
)
a.execute()
# We copied the file into the directory, but since in the container the
# file was called "file" we'll see it with that name in tempDir
self.assertEquals(1, len(os.listdir(tempDir)))
with open(os.path.join(tempDir, 'file')) as f:
data = f.read()
self.assertEquals('data', data)
# Cleanup
os.unlink(tempFile)
shutil.rmtree(tempDir)
def assertMsgIsCorrect(msg, command):
a = DockerApp('a', 'a', image='ubuntu:14.04', command=command)
b = FileDROP('b','b')
a.addOutput(b)
with DROPWaiterCtx(self, b, 100):
a.execute()
self.assertEquals(msg, droputils.allDropContents(b))
def test_additional_bindings(self):
# Some additional stuff to bind into docker
tempDir = tempfile.mkdtemp()
tempFile = tempfile.mktemp()
with open(tempFile, 'w') as f:
f.write('data')
# One binding specifies the target path in the container, the other doesn't
# so it defaults to the same path
a = DockerApp(
'a',
'a',
image='ubuntu:14.04',
command="cp /opt/file %s" % (tempDir, ),
additionalBindings=[tempDir,
"%s:/opt/file" % (tempFile, )])
a.execute()
# We copied the file into the directory, but since in the container the
# file was called "file" we'll see it with that name in tempDir
self.assertEqual(1, len(os.listdir(tempDir)))
with open(os.path.join(tempDir, 'file')) as f:
data = f.read()
self.assertEqual('data', data)
# Cleanup
os.unlink(tempFile)
shutil.rmtree(tempDir)
def _ngas_and_fs_io(self, command):
a = NgasDROP('a', 'a') # not a filesystem-related DROP, we can reference its URL in the command-line
b = DockerApp('b', 'b', image="ubuntu:14.04", command=command)
c = FileDROP('c', 'c')
b.addInput(a)
b.addOutput(c)
with DROPWaiterCtx(self, b, 100):
a.setCompleted()
self.assertEquals(a.dataURL, droputils.allDropContents(c))
def assertMsgIsCorrect(msg, command):
a = DockerApp('a', 'a', image='ubuntu:14.04', command=command)
b = FileDROP('b', 'b')
a.addOutput(b)
with DROPWaiterCtx(self, b, 100):
a.execute()
self.assertEqual(six.b(msg), droputils.allDropContents(b))
def test_start_graph(self):
s3_drop = MockS3DROP(self.get_oid('s3'), uuid.uuid4(), bucket='mock', key='key123', profile_name='aws-profile')
copy_from_s3 = DockerApp(self.get_oid('app'), uuid.uuid4(), image='mock:latest', command='copy_from_s3.sh %iDataURL0 /dfms_root/%o0', user='******')
measurement_set = DirectoryContainer(self.get_oid('dir'), uuid.uuid4(), dirname=TestChiles02._temp)
copy_from_s3.addInput(s3_drop)
copy_from_s3.addOutput(measurement_set)
with droputils.DROPWaiterCtx(self, measurement_set, 50000):
s3_drop.setCompleted()
def test_simpleCopy(self):
"""
Simple test for a dockerized application. It copies the contents of one
file into another via the command-line cp utility. It then checks that
the contents of the target DROP are correct, and that the target file is
actually owned by our process.
The test will not run if a docker daemon cannot be contacted though;
this is to avoid failures in machines that don't have a docker service
running.
"""
try:
AutoVersionClient()
except DockerException:
warnings.warn("Cannot contact the Docker daemon, skipping docker tests")
return
a = FileDROP('a', 'a')
b = DockerApp('b', 'b', image='ubuntu:14.04', command='cp %i0 %o0')
c = FileDROP('c', 'c')
b.addInput(a)
b.addOutput(c)
# Random data so we always check different contents
data = ''.join([random.choice(string.ascii_letters + string.digits) for _ in xrange(10)])
with DROPWaiterCtx(self, c, 100):
a.write(data)
a.setCompleted()
self.assertEquals(data, droputils.allDropContents(c))
# We own the file, not root
uid = os.getuid()
self.assertEquals(uid, os.stat(c.path).st_uid)
def _ngas_and_fs_io(self, command):
a = NgasDROP(
'a', 'a'
) # not a filesystem-related DROP, we can reference its URL in the command-line
b = DockerApp('b', 'b', image="ubuntu:14.04", command=command)
c = FileDROP('c', 'c')
b.addInput(a)
b.addOutput(c)
with DROPWaiterCtx(self, b, 100):
a.setCompleted()
self.assertEqual(six.b(a.dataURL), droputils.allDropContents(c))
def test_simpleCopy(self):
"""
Simple test for a dockerized application. It copies the contents of one
file into another via the command-line cp utility. It then checks that
the contents of the target DROP are correct, and that the target file is
actually owned by our process.
The test will not run if a docker daemon cannot be contacted though;
this is to avoid failures in machines that don't have a docker service
running.
"""
try:
AutoVersionClient().close()
except DockerException:
warnings.warn(
"Cannot contact the Docker daemon, skipping docker tests")
return
a = FileDROP('a', 'a')
b = DockerApp('b', 'b', image='ubuntu:14.04', command='cp %i0 %o0')
c = FileDROP('c', 'c')
b.addInput(a)
b.addOutput(c)
# Random data so we always check different contents
data = os.urandom(10)
with DROPWaiterCtx(self, c, 100):
a.write(data)
a.setCompleted()
self.assertEqual(data, droputils.allDropContents(c))
# We own the file, not root
uid = os.getuid()
self.assertEqual(uid, os.stat(c.path).st_uid)
def test_clientServer(self):
"""
A client-server duo. The server outputs the data it receives to its
output DROP, which in turn is the data held in its input DROP. The graph
looks like this:
A --|--> B(client) --|--> D
|--> C(server) --|
C is a server application which B connects to. Therefore C must be
started before B, so B knows C's IP address and connects successfully.
Although the real writing is done by C, B in this example is also
treated as a publisher of D. This way D waits for both applications to
finish before proceeding.
"""
try:
AutoVersionClient()
except DockerException:
warnings.warn("Cannot contact the Docker daemon, skipping docker tests")
return
a = FileDROP('a', 'a')
b = DockerApp('b', 'b', image='ubuntu:14.04', command='cat %i0 > /dev/tcp/%containerIp[c]%/8000')
c = DockerApp('c', 'c', image='ubuntu:14.04', command='nc -l 8000 > %o0')
d = FileDROP('d', 'd')
b.addInput(a)
b.addOutput(d)
c.addInput(a)
c.addOutput(d)
# Let 'b' handle its interest in c
b.handleInterest(c)
data = ''.join([random.choice(string.ascii_letters + string.digits) for _ in xrange(10)])
with DROPWaiterCtx(self, d, 100):
a.write(data)
a.setCompleted()
self.assertEquals(data, droputils.allDropContents(d))
def test_clientServer(self):
"""
A client-server duo. The server outputs the data it receives to its
output DROP, which in turn is the data held in its input DROP. The graph
looks like this:
A --|--> B(client) --|--> D
|--> C(server) --|
C is a server application which B connects to. Therefore C must be
started before B, so B knows C's IP address and connects successfully.
Although the real writing is done by C, B in this example is also
treated as a publisher of D. This way D waits for both applications to
finish before proceeding.
"""
try:
AutoVersionClient().close()
except DockerException:
warnings.warn(
"Cannot contact the Docker daemon, skipping docker tests")
return
a = FileDROP('a', 'a')
b = DockerApp('b',
'b',
image='ubuntu:14.04',
command='cat %i0 > /dev/tcp/%containerIp[c]%/8000')
c = DockerApp('c',
'c',
image='ubuntu:14.04',
command='nc -l 8000 > %o0')
d = FileDROP('d', 'd')
b.addInput(a)
b.addOutput(d)
c.addInput(a)
c.addOutput(d)
# Let 'b' handle its interest in c
b.handleInterest(c)
data = os.urandom(10)
with DROPWaiterCtx(self, d, 100):
a.write(data)
a.setCompleted()
self.assertEqual(data, droputils.allDropContents(d))
def test_to_first_split(self):
s3_drop = MockS3DROP(self.get_oid('s3'), uuid.uuid4(), bucket='mock', key='key123', profile_name='aws-profile')
copy_from_s3 = DockerApp(self.get_oid('app'), uuid.uuid4(), image='mock:latest', command='copy_from_s3.sh %iDataURL0 /dfms_root/%o0', user='******')
measurement_set = DirectoryContainer(self.get_oid('dir'), uuid.uuid4(), dirname=TestChiles02._temp)
copy_from_s3.addInput(s3_drop)
copy_from_s3.addOutput(measurement_set)
outputs = []
frequencies = make_groups_of_frequencies(FREQUENCY_GROUPS, 5)
frequencies = frequencies[0]
for group in frequencies:
casa_py_drop = DockerApp(self.get_oid('app'), uuid.uuid4(), image='mock:latest', command='casa_py.sh /dfms_root/%i0 /dfms_root/%o0 {0} {1}'.format(group[0], group[1]), user='******')
result = FileDROP(self.get_oid('file'), uuid.uuid4(), dirname=TestChiles02._temp)
copy_to_s3 = DockerApp(self.get_oid('app'), uuid.uuid4(), image='mock:latest', command='copy_to_s3.sh /dfms_root/%i0 %oDataURL0', user='******')
s3_drop_out = MockS3DROP(self.get_oid('s3'), uuid.uuid4(), bucket='mock', key='{0}_{1}/key123'.format(group[0], group[1]), profile_name='aws-profile')
casa_py_drop.addInput(measurement_set)
casa_py_drop.addOutput(result)
copy_to_s3.addInput(result)
copy_to_s3.addOutput(s3_drop_out)
outputs.append(s3_drop_out)
barrier_drop = BarrierAppDROP(self.get_oid('barrier'), uuid.uuid4())
barrier_drop.addInput(measurement_set)
for output in outputs:
barrier_drop.addInput(output)
with droputils.DROPWaiterCtx(self, barrier_drop, 50000):
s3_drop.setCompleted()
