def test_to_first_split(self):
s3_drop = MockS3DROP(self.get_oid('s3'), uuid.uuid4(), bucket='mock', key='key123', profile_name='aws-profile')
copy_from_s3 = DockerApp(self.get_oid('app'), uuid.uuid4(), image='mock:latest', command='copy_from_s3.sh %iDataURL0 /dfms_root/%o0', user='******')
measurement_set = DirectoryContainer(self.get_oid('dir'), uuid.uuid4(), dirname=TestChiles02._temp)
copy_from_s3.addInput(s3_drop)
copy_from_s3.addOutput(measurement_set)
outputs = []
frequencies = make_groups_of_frequencies(FREQUENCY_GROUPS, 5)
frequencies = frequencies[0]
for group in frequencies:
casa_py_drop = DockerApp(self.get_oid('app'), uuid.uuid4(), image='mock:latest', command='casa_py.sh /dfms_root/%i0 /dfms_root/%o0 {0} {1}'.format(group[0], group[1]), user='******')
result = FileDROP(self.get_oid('file'), uuid.uuid4(), dirname=TestChiles02._temp)
copy_to_s3 = DockerApp(self.get_oid('app'), uuid.uuid4(), image='mock:latest', command='copy_to_s3.sh /dfms_root/%i0 %oDataURL0', user='******')
s3_drop_out = MockS3DROP(self.get_oid('s3'), uuid.uuid4(), bucket='mock', key='{0}_{1}/key123'.format(group[0], group[1]), profile_name='aws-profile')
casa_py_drop.addInput(measurement_set)
casa_py_drop.addOutput(result)
copy_to_s3.addInput(result)
copy_to_s3.addOutput(s3_drop_out)
outputs.append(s3_drop_out)
barrier_drop = BarrierAppDROP(self.get_oid('barrier'), uuid.uuid4())
barrier_drop.addInput(measurement_set)
for output in outputs:
barrier_drop.addInput(output)
with droputils.DROPWaiterCtx(self, barrier_drop, 50000):
s3_drop.setCompleted()
def test_start_graph(self):
s3_drop = MockS3DROP(self.get_oid('s3'), uuid.uuid4(), bucket='mock', key='key123', profile_name='aws-profile')
copy_from_s3 = DockerApp(self.get_oid('app'), uuid.uuid4(), image='mock:latest', command='copy_from_s3.sh %iDataURL0 /dfms_root/%o0', user='******')
measurement_set = DirectoryContainer(self.get_oid('dir'), uuid.uuid4(), dirname=TestChiles02._temp)
copy_from_s3.addInput(s3_drop)
copy_from_s3.addOutput(measurement_set)
with droputils.DROPWaiterCtx(self, measurement_set, 50000):
s3_drop.setCompleted()
def test_many_relationships(self):
"""
A test in which a drop is related to many other drops that live in a
separate DM.
Drop A is accessed by many applications (B1, B2, .., BN), which should
not exhaust resources on DM #1. We collapse all into C so we can monitor
only its status to know that the execution is over.
DM #1 DM #2
======= ====================
| | | |--> B1 --| |
| | | |--> B2 --| |
| A --|----|-|--> B3 --|--> C |
| | | |.........| |
| | | |--> BN --| |
======= ====================
"""
dm1, dm2 = [self._start_dm() for _ in range(2)]
sessionId = 's1'
N = 100
g1 = [{"oid": "A", "type": "plain", "storage": "memory"}]
g2 = [{"oid": "C", "type": "plain", "storage": "memory"}]
rels = []
for i in range(N):
b_oid = "B%d" % (i, )
# SleepAndCopyApp effectively opens the input drop
g2.append({
"oid": b_oid,
"type": "app",
"app": "test.graphsRepository.SleepAndCopyApp",
"outputs": ["C"],
"sleepTime": 0
})
rels.append(DROPRel('A', DROPLinkType.INPUT, b_oid))
quickDeploy(dm1, sessionId, g1, {nm_conninfo(1): rels})
quickDeploy(dm2, sessionId, g2, {nm_conninfo(0): rels})
self.assertEqual(1, len(dm1._sessions[sessionId].drops))
self.assertEqual(1 + N, len(dm2._sessions[sessionId].drops))
# Run! The sole fact that this doesn't throw exceptions is already
# a good proof that everything is working as expected
a = dm1._sessions[sessionId].drops['A']
c = dm2._sessions[sessionId].drops['C']
with droputils.DROPWaiterCtx(self, c, 10):
a.write('a')
a.setCompleted()
for i in range(N):
drop = dm2._sessions[sessionId].drops["B%d" % (i, )]
self.assertEqual(DROPStates.COMPLETED, drop.status)
dm1.destroySession(sessionId)
dm2.destroySession(sessionId)
def test_runGraphOneDOPerDOM(self):
"""
A test that creates three DROPs in two different DMs and runs the graph.
For this the graphs that are fed into the DMs must *not* express the
inter-DM relationships, although they are still passed down
separately. The graph looks like:
DM #1 DM #2
======= =============
| A --|----|-> B --> C |
======= =============
"""
dm1, dm2 = [self._start_dm() for _ in range(2)]
sessionId = 's1'
g1 = [{"oid": "A", "type": "plain", "storage": "memory"}]
g2 = [{
"oid": "B",
"type": "app",
"app": "dfms.apps.crc.CRCApp"
}, {
"oid": "C",
"type": "plain",
"storage": "memory",
"producers": ["B"]
}]
rels = [DROPRel('B', DROPLinkType.CONSUMER, 'A')]
quickDeploy(dm1, sessionId, g1, {nm_conninfo(1): rels})
quickDeploy(dm2, sessionId, g2, {nm_conninfo(0): rels})
self.assertEqual(1, len(dm1._sessions[sessionId].drops))
self.assertEqual(2, len(dm2._sessions[sessionId].drops))
# Run! We wait until c is completed
a = dm1._sessions[sessionId].drops['A']
b, c = [dm2._sessions[sessionId].drops[x] for x in ('B', 'C')]
with droputils.DROPWaiterCtx(self, c, 1):
a.write('a')
a.setCompleted()
for drop in a, b, c:
self.assertEqual(DROPStates.COMPLETED, drop.status)
self.assertEqual(a.checksum, int(droputils.allDropContents(c)))
dm1.destroySession(sessionId)
dm2.destroySession(sessionId)
def test_sql_create(self):
sqlite01 = get_oid('sqlite')
sqlite_drop = FileDROP(
sqlite01,
get_uuid(),
precious=False,
dirname=os.path.join(TestSqlite._temp, sqlite01),
check_exists=False,
)
initialize_sqlite = InitializeSqliteApp(
get_oid('app'),
get_uuid(),
user='******',
)
sqlite_in_memory = InMemoryDROP(
get_oid('memory'),
get_uuid(),
precious=False,
)
initialize_sqlite.addInput(sqlite_drop)
initialize_sqlite.addOutput(sqlite_in_memory)
with droputils.DROPWaiterCtx(self, sqlite_in_memory, 50000):
sqlite_drop.setCompleted()
def test_runWithFourDMs(self):
"""
A test that creates several DROPs in two different DMs and runs
the graph. The graph looks like this
DM #2
+--------------------------+
| |--> C --| |
+---|--> B --|--> D --|--> F --|--|
| | |--> E --| | |
DM #1 | +--------------------------+ | DM #4
+-----+ | | +---------------------+
| | | |--|--> L --| |
| A --|--+ | |--> N --> O |
| | | |--|--> M --| |
+-----+ | DM #3 | +---------------------+
| +--------------------------+ |
| | |--> H --| | |
+---|--> G --|--> I --|--> K --|--|
| |--> J --| |
+--------------------------+
B, F, G, K and N are AppDOs; the rest are plain in-memory DROPs
"""
dm1, dm2, dm3, dm4 = [self._start_dm() for _ in range(4)]
sessionId = 's1'
g1 = [memory('A', expectedSize=1)]
g2 = [
sleepAndCopy('B', outputs=['C', 'D', 'E'], sleepTime=0),
memory('C'),
memory('D'),
memory('E'),
sleepAndCopy('F', inputs=['C', 'D', 'E'], sleepTime=0)
]
g3 = [
sleepAndCopy('G', outputs=['H', 'I', 'J'], sleepTime=0),
memory('H'),
memory('I'),
memory('J'),
sleepAndCopy('K', inputs=['H', 'I', 'J'], sleepTime=0)
]
g4 = [
memory('L'),
memory('M'),
sleepAndCopy('N', inputs=['L', 'M'], outputs=['O'], sleepTime=0),
memory('O')
]
rels_12 = [DROPRel('A', DROPLinkType.INPUT, 'B')]
rels_13 = [DROPRel('A', DROPLinkType.INPUT, 'G')]
rels_24 = [DROPRel('F', DROPLinkType.PRODUCER, 'L')]
rels_34 = [DROPRel('K', DROPLinkType.PRODUCER, 'M')]
quickDeploy(dm1, sessionId, g1, {
nm_conninfo(1): rels_12,
nm_conninfo(2): rels_13
})
quickDeploy(dm2, sessionId, g2, {
nm_conninfo(0): rels_12,
nm_conninfo(3): rels_24
})
quickDeploy(dm3, sessionId, g3, {
nm_conninfo(0): rels_13,
nm_conninfo(3): rels_34
})
quickDeploy(dm4, sessionId, g4, {
nm_conninfo(1): rels_24,
nm_conninfo(2): rels_34
})
self.assertEqual(1, len(dm1._sessions[sessionId].drops))
self.assertEqual(5, len(dm2._sessions[sessionId].drops))
self.assertEqual(5, len(dm3._sessions[sessionId].drops))
self.assertEqual(4, len(dm4._sessions[sessionId].drops))
a = dm1._sessions[sessionId].drops['A']
o = dm4._sessions[sessionId].drops['O']
drops = []
for x in (dm1, dm2, dm3, dm4):
drops += x._sessions[sessionId].drops.values()
# Run! This should trigger the full execution of the graph
with droputils.DROPWaiterCtx(self, o, 5):
a.write('a')
for drop in drops:
self.assertEqual(
DROPStates.COMPLETED, drop.status,
"Status of '%s' is not COMPLETED: %d" %
(drop.uid, drop.status))
for dm in [dm1, dm2, dm3, dm4]:
dm.destroySession(sessionId)
def test_runGraphSeveralDropsPerDM(self):
"""
A test that creates several DROPs in two different DMs and runs
the graph. The graph looks like this
DM #1 DM #2
=================== ================
| A --> C --> D --|----|-| |
| | | |--> E --> F |
| B --------------|----|-| |
=================== ================
:see: `self.test_runGraphSingleDOPerDOM`
"""
dm1, dm2 = [self._start_dm() for _ in range(2)]
sessionId = 's1'
g1 = [{
"oid": "A",
"type": "plain",
"storage": "memory",
"consumers": ["C"]
}, {
"oid": "B",
"type": "plain",
"storage": "memory"
}, {
"oid": "C",
"type": "app",
"app": "dfms.apps.crc.CRCApp"
}, {
"oid": "D",
"type": "plain",
"storage": "memory",
"producers": ["C"]
}]
g2 = [{
"oid": "E",
"type": "app",
"app": "test.test_drop.SumupContainerChecksum"
}, {
"oid": "F",
"type": "plain",
"storage": "memory",
"producers": ["E"]
}]
rels = [
DROPRel('D', DROPLinkType.INPUT, 'E'),
DROPRel('B', DROPLinkType.INPUT, 'E')
]
quickDeploy(dm1, sessionId, g1, {nm_conninfo(1): rels})
quickDeploy(dm2, sessionId, g2, {nm_conninfo(0): rels})
self.assertEqual(4, len(dm1._sessions[sessionId].drops))
self.assertEqual(2, len(dm2._sessions[sessionId].drops))
# Run! The sole fact that this doesn't throw exceptions is already
# a good proof that everything is working as expected
a, b, c, d = [
dm1._sessions[sessionId].drops[x] for x in ('A', 'B', 'C', 'D')
]
e, f = [dm2._sessions[sessionId].drops[x] for x in ('E', 'F')]
with droputils.DROPWaiterCtx(self, f, 5):
a.write('a')
a.setCompleted()
b.write('a')
b.setCompleted()
for drop in a, b, c, d, e, f:
self.assertEqual(DROPStates.COMPLETED, drop.status,
"DROP %s is not COMPLETED" % (drop.uid))
self.assertEqual(a.checksum, int(droputils.allDropContents(d)))
self.assertEqual(b.checksum + d.checksum,
int(droputils.allDropContents(f)))
dm1.destroySession(sessionId)
dm2.destroySession(sessionId)