def test_dropWroteFromOutside(self):
"""
A different scenario to those tested above, in which the data
represented by the DROP isn't actually written *through* the
DROP. Still, the DROP needs to be moved to COMPLETED once
the data is written, and reading from it should still yield a correct
result
"""
# Write, but not through the DROP
a = FileDROP("A", "A")
filename = a.path
msg = "a message"
with open(filename, "w") as f:
f.write(msg)
a.setCompleted()
# Read from the DROP
self.assertEquals(msg, droputils.allDropContents(a))
self.assertIsNone(a.checksum)
self.assertIsNone(a.size)
# We can manually set the size because the DROP wasn't able to calculate
# it itself; if we couldn't an exception would be thrown
a.size = len(msg)
def test_dropWroteFromOutside(self):
"""
A different scenario to those tested above, in which the data
represented by the DROP isn't actually written *through* the
DROP. Still, the DROP needs to be moved to COMPLETED once
the data is written, and reading from it should still yield a correct
result
"""
# Write, but not through the DROP
a = FileDROP('A', 'A')
filename = a.path
msg = 'a message'
with open(filename, 'w') as f:
f.write(msg)
a.setCompleted()
# Read from the DROP
self.assertEqual(six.b(msg), droputils.allDropContents(a))
self.assertIsNone(a.checksum)
self.assertIsNone(a.size)
# We can manually set the size because the DROP wasn't able to calculate
# it itself; if we couldn't an exception would be thrown
a.size = len(msg)
def assertFiles(delete_parent_directory, parentDirExists, tempDir=None):
tempDir = tempDir or tempfile.mkdtemp()
a = FileDROP("a", "a", dirname=tempDir, delete_parent_directory=delete_parent_directory)
a.write(" ")
a.setCompleted()
self.assertTrue(a.exists())
self.assertTrue(os.path.isdir(tempDir))
a.delete()
self.assertFalse(a.exists())
self.assertEquals(parentDirExists, os.path.isdir(tempDir))
if parentDirExists:
shutil.rmtree(tempDir)
def assertFiles(delete_parent_directory, parentDirExists, tempDir=None):
tempDir = tempDir or tempfile.mkdtemp()
a = FileDROP('a', 'a', dirname=tempDir, delete_parent_directory=delete_parent_directory)
a.write(' ')
a.setCompleted()
self.assertTrue(a.exists())
self.assertTrue(os.path.isdir(tempDir))
a.delete()
self.assertFalse(a.exists())
self.assertEqual(parentDirExists, os.path.isdir(tempDir))
if parentDirExists:
shutil.rmtree(tempDir)
def test_clientServer(self):
"""
A client-server duo. The server outputs the data it receives to its
output DROP, which in turn is the data held in its input DROP. The graph
looks like this:
A --|--> B(client) --|--> D
|--> C(server) --|
C is a server application which B connects to. Therefore C must be
started before B, so B knows C's IP address and connects successfully.
Although the real writing is done by C, B in this example is also
treated as a publisher of D. This way D waits for both applications to
finish before proceeding.
"""
try:
AutoVersionClient().close()
except DockerException:
warnings.warn(
"Cannot contact the Docker daemon, skipping docker tests")
return
a = FileDROP('a', 'a')
b = DockerApp('b',
'b',
image='ubuntu:14.04',
command='cat %i0 > /dev/tcp/%containerIp[c]%/8000')
c = DockerApp('c',
'c',
image='ubuntu:14.04',
command='nc -l 8000 > %o0')
d = FileDROP('d', 'd')
b.addInput(a)
b.addOutput(d)
c.addInput(a)
c.addOutput(d)
# Let 'b' handle its interest in c
b.handleInterest(c)
data = os.urandom(10)
with DROPWaiterCtx(self, d, 100):
a.write(data)
a.setCompleted()
self.assertEqual(data, droputils.allDropContents(d))
def test_echo(self):
a = FileDROP('a', 'a')
b = BashShellApp('b', 'b', command='cp %i0 %o0')
c = FileDROP('c', 'c')
b.addInput(a)
b.addOutput(c)
# Random data so we always check different contents
data = ''.join([random.choice(string.ascii_letters + string.digits) for _ in xrange(10)])
with DROPWaiterCtx(self, c, 100):
a.write(data)
a.setCompleted()
self.assertEquals(data, droputils.allDropContents(c))
# We own the file, not root
uid = os.getuid()
self.assertEquals(uid, os.stat(c.path).st_uid)
def test_echo(self):
a = FileDROP('a', 'a')
b = BashShellApp('b', 'b', command='cp %i0 %o0')
c = FileDROP('c', 'c')
b.addInput(a)
b.addOutput(c)
# Random data so we always check different contents
data = os.urandom(10)
with DROPWaiterCtx(self, c, 100):
a.write(data)
a.setCompleted()
self.assertEqual(data, droputils.allDropContents(c))
# We own the file, not root
uid = os.getuid()
self.assertEqual(uid, os.stat(c.path).st_uid)
def test_clientServer(self):
"""
A client-server duo. The server outputs the data it receives to its
output DROP, which in turn is the data held in its input DROP. The graph
looks like this:
A --|--> B(client) --|--> D
|--> C(server) --|
C is a server application which B connects to. Therefore C must be
started before B, so B knows C's IP address and connects successfully.
Although the real writing is done by C, B in this example is also
treated as a publisher of D. This way D waits for both applications to
finish before proceeding.
"""
try:
AutoVersionClient()
except DockerException:
warnings.warn("Cannot contact the Docker daemon, skipping docker tests")
return
a = FileDROP('a', 'a')
b = DockerApp('b', 'b', image='ubuntu:14.04', command='cat %i0 > /dev/tcp/%containerIp[c]%/8000')
c = DockerApp('c', 'c', image='ubuntu:14.04', command='nc -l 8000 > %o0')
d = FileDROP('d', 'd')
b.addInput(a)
b.addOutput(d)
c.addInput(a)
c.addOutput(d)
# Let 'b' handle its interest in c
b.handleInterest(c)
data = ''.join([random.choice(string.ascii_letters + string.digits) for _ in xrange(10)])
with DROPWaiterCtx(self, d, 100):
a.write(data)
a.setCompleted()
self.assertEquals(data, droputils.allDropContents(d))
def test_simpleCopy(self):
"""
Simple test for a dockerized application. It copies the contents of one
file into another via the command-line cp utility. It then checks that
the contents of the target DROP are correct, and that the target file is
actually owned by our process.
The test will not run if a docker daemon cannot be contacted though;
this is to avoid failures in machines that don't have a docker service
running.
"""
try:
AutoVersionClient().close()
except DockerException:
warnings.warn(
"Cannot contact the Docker daemon, skipping docker tests")
return
a = FileDROP('a', 'a')
b = DockerApp('b', 'b', image='ubuntu:14.04', command='cp %i0 %o0')
c = FileDROP('c', 'c')
b.addInput(a)
b.addOutput(c)
# Random data so we always check different contents
data = os.urandom(10)
with DROPWaiterCtx(self, c, 100):
a.write(data)
a.setCompleted()
self.assertEqual(data, droputils.allDropContents(c))
# We own the file, not root
uid = os.getuid()
self.assertEqual(uid, os.stat(c.path).st_uid)
def test_simpleCopy(self):
"""
Simple test for a dockerized application. It copies the contents of one
file into another via the command-line cp utility. It then checks that
the contents of the target DROP are correct, and that the target file is
actually owned by our process.
The test will not run if a docker daemon cannot be contacted though;
this is to avoid failures in machines that don't have a docker service
running.
"""
try:
AutoVersionClient()
except DockerException:
warnings.warn("Cannot contact the Docker daemon, skipping docker tests")
return
a = FileDROP('a', 'a')
b = DockerApp('b', 'b', image='ubuntu:14.04', command='cp %i0 %o0')
c = FileDROP('c', 'c')
b.addInput(a)
b.addOutput(c)
# Random data so we always check different contents
data = ''.join([random.choice(string.ascii_letters + string.digits) for _ in xrange(10)])
with DROPWaiterCtx(self, c, 100):
a.write(data)
a.setCompleted()
self.assertEquals(data, droputils.allDropContents(c))
# We own the file, not root
uid = os.getuid()
self.assertEquals(uid, os.stat(c.path).st_uid)
def test_sql_create(self):
sqlite01 = get_oid('sqlite')
sqlite_drop = FileDROP(
sqlite01,
get_uuid(),
precious=False,
dirname=os.path.join(TestSqlite._temp, sqlite01),
check_exists=False,
)
initialize_sqlite = InitializeSqliteApp(
get_oid('app'),
get_uuid(),
user='******',
)
sqlite_in_memory = InMemoryDROP(
get_oid('memory'),
get_uuid(),
precious=False,
)
initialize_sqlite.addInput(sqlite_drop)
initialize_sqlite.addOutput(sqlite_in_memory)
with droputils.DROPWaiterCtx(self, sqlite_in_memory, 50000):
sqlite_drop.setCompleted()
