forked from distributed-system-analysis/smallfile
-
Notifications
You must be signed in to change notification settings - Fork 0
/
invoke_process.py
130 lines (115 loc) · 4.82 KB
/
invoke_process.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
'''
invoke_process.py -- launch multiple subprocesses, each running smf_invocation instance
Copyright 2012 -- Ben England
Licensed under the Apache License at http://www.apache.org/licenses/LICENSE-2.0
See Appendix on this page for instructions pertaining to license.
'''
import multiprocessing
import unittest
import smallfile
import os
import time
OK = 0 # system call return code for success
# this class is used to launch multiple threads with smf_invocation instances
# we do this because we can use > 1 core this way, with
# python threading, it doesn't really use > 1 core because of the GIL (global lock)
# occasional status reports can be sent back using pipe as well
class subprocess(multiprocessing.Process):
def __init__(self, invocation):
multiprocessing.Process.__init__(self)
(conn1, conn2) = multiprocessing.Pipe(False)
self.receiver = conn1 # master process receives data on results of test here
self.sender = conn2 # slave process sends data on results of test here
self.invoke = invocation # all the workload generation is done by this object
def run(self):
try:
self.invoke.do_workload()
self.invoke.log.debug('exiting subprocess and returning invoke '+ str(self.invoke))
except Exception as e:
print 'Exception seen in thread %s (tail /var/tmp/invoke_logs_%s.log) '%(self.invoke.tid, self.invoke.tid)
self.invoke.log.error(str(e))
self.status = self.invoke.NOTOK
finally:
self.rsptimes = None # response time array should have been saved to file first
self.invoke.log = None # log objects cannot be serialized
self.sender.send(self.invoke)
# below are unit tests for smf_invocation
# including multi-threaded test
ok=0
class Test(unittest.TestCase):
topdir='/var/tmp/subproctest'
def setUp(self):
self.invok = smallfile.smf_invocation()
self.invok.debug = True
self.invok.verbose = True
self.invok.tid = "regtest"
self.invok.tmp_dir = self.topdir
self.invok.src_dir = self.topdir + os.sep + 'src'
self.invok.dest_dir = self.topdir + os.sep + 'dst'
self.invok.start_log()
def deltree(self):
self.invok.log.debug('deleting entire %s'%self.topdir)
if not os.path.exists(self.topdir): return
if not os.path.isdir(self.topdir): return
for (dir, subdirs, files) in os.walk(self.topdir, topdown=False):
for f in files: os.unlink(os.path.join(dir,f))
for d in subdirs: os.rmdir(os.path.join(dir,d))
os.rmdir(self.topdir)
def chk_status(self):
assert self.invok.status == ok
def runTest(self, opName):
self.invok.opname = opName
self.invok.do_workload()
self.chk_status()
def checkDirEmpty(self, emptyDir):
self.assertTrue(len(glob.glob(emptyDir + os.sep + "*")) == 0)
def test_multiproc_stonewall(self):
self.invok.start_log()
self.invok.log.info('starting stonewall test')
thread_ready_timeout = 4
thread_count = 4
self.deltree()
os.mkdir(self.topdir)
os.mkdir(self.invok.src_dir)
os.mkdir(self.invok.dest_dir)
sgate_file = self.invok.tmp_dir + os.sep + "start"
smallfile.ensure_deleted(sgate_file)
invokeList = []
for j in range(0, thread_count):
s = smallfile.smf_invocation()
#s.log_to_stderr = True
s.verbose = True
s.tid = str(j)
s.src_dir = self.invok.src_dir
s.dst_dir = self.invok.dest_dir
s.prefix = "thr_"
s.suffix = "foo"
s.iterations=10
s.stonewall = True
s.starting_gate = sgate_file
s.do_sync_at_end = True;
invokeList.append(s)
threadList=[]
for s in invokeList: threadList.append(subprocess(s))
for t in threadList: t.start()
threads_ready = True
for i in range(0, thread_ready_timeout):
threads_ready = True
for s in invokeList:
thread_ready_file = s.gen_thread_ready_fname(s.tid)
if not os.access(thread_ready_file, os.R_OK): threads_ready = False
if threads_ready: break
time.sleep(1)
if not threads_ready: raise Exception("threads did not show up within %d seconds"%thread_ready_timeout)
time.sleep(1)
f = open(sgate_file, "w")
f.close()
for t in threadList:
rtnd_invok = t.receiver.recv()
t.join()
self.invok.log.info(str(rtnd_invok))
if rtnd_invok.status != ok:
raise Exception("subprocess failure: " + str(t))
# so you can just do "python invoke_process.py" to test it
if __name__ == "__main__":
unittest.main()