def test_get_host_from_id(self):
b = barrier.barrier('127.0.0.1#', 'testgethost', 100)
hostname = b.get_host_from_id('my_host')
self.assertEqual(hostname, 'my_host')
hostname = b.get_host_from_id('my_host#')
self.assertEqual(hostname, 'my_host')
self.assertRaises(error.BarrierError, b.get_host_from_id, '#my_host')
def _rdv(addr):
b1 = barrier.barrier(addr, "test_meeting", timeout, port)
if not rendezvous_servers:
if test_abort:
b1.rendezvous('127.0.0.1#0', '127.0.0.1#1', abort=abort)
else:
b1.rendezvous('127.0.0.1#0', '127.0.0.1#1')
else:
if test_abort:
b1.rendezvous_servers('127.0.0.1#0', '127.0.0.1#1',
abort=abort)
else:
b1.rendezvous_servers('127.0.0.1#0', '127.0.0.1#1')
def _rdv(addr):
b1 = barrier.barrier(addr,
"test_meeting",
timeout,
port,
listen_server=listen_server)
if not rendezvous_servers:
if test_abort:
b1.rendezvous('127.0.0.1#0', '127.0.0.1#1', abort=abort)
else:
b1.rendezvous('127.0.0.1#0', '127.0.0.1#1')
else:
if test_abort:
b1.rendezvous_servers('127.0.0.1#0',
'127.0.0.1#1',
abort=abort)
else:
b1.rendezvous_servers('127.0.0.1#0', '127.0.0.1#1')
def test_master_welcome_garbage(self):
b = barrier.barrier('127.0.0.1#', 'garbage', 100)
waiting_before = dict(b._waiting)
seen_before = b._seen
sender, receiver = socket.socketpair()
try:
sender.send('GET /foobar?p=-1 HTTP/1.0\r\n\r\n')
# This should not raise an exception.
b._master_welcome((receiver, 'fakeaddr'))
self.assertEqual(waiting_before, b._waiting)
self.assertEqual(seen_before, b._seen)
sender, receiver = socket.socketpair()
sender.send('abcdefg\x00\x01\x02\n' * 5)
# This should not raise an exception.
b._master_welcome((receiver, 'fakeaddr'))
self.assertEqual(waiting_before, b._waiting)
self.assertEqual(seen_before, b._seen)
finally:
sender.close()
receiver.close()
def test_master_welcome_garbage(self):
b = barrier.barrier('127.0.0.1#', 'garbage', 100)
waiting_before = dict(b.waiting)
seen_before = b.seen
sender, receiver = socket.socketpair()
try:
sender.send('GET /foobar?p=-1 HTTP/1.0\r\n\r\n')
# This should not raise an exception.
b.master_welcome((receiver, 'fakeaddr'))
self.assertEqual(waiting_before, b.waiting)
self.assertEqual(seen_before, b.seen)
sender, receiver = socket.socketpair()
sender.send('abcdefg\x00\x01\x02\n'*5)
# This should not raise an exception.
b.master_welcome((receiver, 'fakeaddr'))
self.assertEqual(waiting_before, b.waiting)
self.assertEqual(seen_before, b.seen)
finally:
sender.close()
receiver.close()
def finish_profilers(machines, timeout=120):
sb = barrier.barrier(_PROF_MASTER, "finish_profilers", timeout, port=_PORT)
sb.rendezvous_servers(_PROF_MASTER, *machines)
def test_update_timeout(self):
b = barrier.barrier('127.0.0.1#', 'update', 100)
b.update_timeout(120)
self.assertEqual(b.timeout, 120)
def get_sync_control_file(control, host_name, host_num,
instance, num_jobs, port_base=63100):
"""
This function is used when there is a need to run more than one
job simultaneously starting exactly at the same time. It basically returns
a modified control file (containing the synchronization code prepended)
whenever it is ready to run the control file. The synchronization
is done using barriers to make sure that the jobs start at the same time.
Here is how the synchronization is done to make sure that the tests
start at exactly the same time on the client.
sc_bar is a server barrier and s_bar, c_bar are the normal barriers
Job1 Job2 ...... JobN
Server: | sc_bar
Server: | s_bar ...... s_bar
Server: | at.run() at.run() ...... at.run()
----------|------------------------------------------------------
Client | sc_bar
Client | c_bar c_bar ...... c_bar
Client | <run test> <run test> ...... <run test>
@param control: The control file which to which the above synchronization
code will be prepended.
@param host_name: The host name on which the job is going to run.
@param host_num: (non negative) A number to identify the machine so that
we have different sets of s_bar_ports for each of the machines.
@param instance: The number of the job
@param num_jobs: Total number of jobs that are going to run in parallel
with this job starting at the same time.
@param port_base: Port number that is used to derive the actual barrier
ports.
@returns The modified control file.
"""
sc_bar_port = port_base
c_bar_port = port_base
if host_num < 0:
print "Please provide a non negative number for the host"
return None
s_bar_port = port_base + 1 + host_num # The set of s_bar_ports are
# the same for a given machine
sc_bar_timeout = 180
s_bar_timeout = c_bar_timeout = 120
# The barrier code snippet is prepended into the conrol file
# dynamically before at.run() is called finally.
control_new = []
# jobid is the unique name used to identify the processes
# trying to reach the barriers
jobid = "%s#%d" % (host_name, instance)
rendv = []
# rendvstr is a temp holder for the rendezvous list of the processes
for n in range(num_jobs):
rendv.append("'%s#%d'" % (host_name, n))
rendvstr = ",".join(rendv)
if instance == 0:
# Do the setup and wait at the server barrier
# Clean up the tmp and the control dirs for the first instance
control_new.append('if os.path.exists(job.tmpdir):')
control_new.append("\t system('umount -f %s > /dev/null"
"2> /dev/null' % job.tmpdir,"
"ignore_status=True)")
control_new.append("\t system('rm -rf ' + job.tmpdir)")
control_new.append(
'b0 = job.barrier("%s", "sc_bar", %d, port=%d)'
% (jobid, sc_bar_timeout, sc_bar_port))
control_new.append(
'b0.rendezvous_servers("PARALLEL_MASTER", "%s")'
% jobid)
elif instance == 1:
# Wait at the server barrier to wait for instance=0
# process to complete setup
b0 = barrier.barrier("PARALLEL_MASTER", "sc_bar", sc_bar_timeout,
port=sc_bar_port)
b0.rendezvous_servers("PARALLEL_MASTER", jobid)
if(num_jobs > 2):
b1 = barrier.barrier(jobid, "s_bar", s_bar_timeout,
port=s_bar_port)
b1.rendezvous(rendvstr)
else:
# For the rest of the clients
b2 = barrier.barrier(jobid, "s_bar", s_bar_timeout, port=s_bar_port)
b2.rendezvous(rendvstr)
# Client side barrier for all the tests to start at the same time
control_new.append('b1 = job.barrier("%s", "c_bar", %d, port=%d)'
% (jobid, c_bar_timeout, c_bar_port))
control_new.append("b1.rendezvous(%s)" % rendvstr)
# Stick in the rest of the control file
control_new.append(control)
return "\n".join(control_new)
def test_initialize(self):
b = barrier.barrier('127.0.0.1#', 'testtag', 100, 11921)
self.assertEqual(b.hostid, '127.0.0.1#')
self.assertEqual(b.tag, 'testtag')
self.assertEqual(b.timeout, 100)
self.assertEqual(b.port, 11921)
def wait_for_profilers(machines, timeout=300):
sb = barrier.barrier(_PROF_MASTER, "sync_profilers",
timeout, port=_PORT)
sb.rendezvous_servers(_PROF_MASTER, *machines)
def finish_profilers(machines, timeout=120):
sb = barrier.barrier(_PROF_MASTER, "finish_profilers",
timeout, port=_PORT)
sb.rendezvous_servers(_PROF_MASTER, *machines)
def wait_for_profilers(machines, timeout = 300):
sb = barrier.barrier("PROF_MASTER", "sync_profilers",
timeout, port=11920)
sb.rendezvous_servers("PROF_MASTER", *machines)
def stop_profilers(machines, timeout = 120):
sb = barrier.barrier("PROF_MASTER", "stop_profilers",
timeout, port=11920)
sb.rendezvous_servers("PROF_MASTER", *machines)
def test_remaining(self):
b = barrier.barrier('127.0.0.1#', 'remain', 100)
remain = b.remaining()
self.assertEqual(remain, 100)
def wait_for_profilers(machines, timeout=300):
sb = barrier.barrier(_PROF_MASTER, "sync_profilers", timeout, port=_PORT)
sb.rendezvous_servers(_PROF_MASTER, *machines)
def test_remaining(self):
b = barrier.barrier('127.0.0.1#', 'remain', 100)
remain = b._remaining()
self.assertEqual(remain, 100)
def test_update_timeout(self):
b = barrier.barrier('127.0.0.1#', 'update', 100)
b._update_timeout(120)
self.assertEqual(b._timeout_secs, 120)
def test_initialize(self):
b = barrier.barrier('127.0.0.1#', 'testtag', 100, 11921)
self.assertEqual(b._hostid, '127.0.0.1#')
self.assertEqual(b._tag, 'testtag')
self.assertEqual(b._timeout_secs, 100)
self.assertEqual(b._port, 11921)
def get_sync_control_file(control, host_name, host_num,
instance, num_jobs, port_base=63100):
sc_bar_port = port_base
c_bar_port = port_base
if host_num < 0:
print "Please provide a non negative number for the host"
return None
s_bar_port = port_base + 1 + host_num # The set of s_bar_ports are
# the same for a given machine
sc_bar_timeout = 180
s_bar_timeout = c_bar_timeout = 120
# The barrier code snippet is prepended into the conrol file
# dynamically before at.run() is called finally.
control_new = []
# jobid is the unique name used to identify the processes
# trying to reach the barriers
jobid = "%s#%d" % (host_name, instance)
rendv = []
# rendvstr is a temp holder for the rendezvous list of the processes
for n in range(num_jobs):
rendv.append("'%s#%d'" % (host_name, n))
rendvstr = ",".join(rendv)
if instance == 0:
# Do the setup and wait at the server barrier
# Clean up the tmp and the control dirs for the first instance
control_new.append('if os.path.exists(job.tmpdir):')
control_new.append("\t system('umount -f %s > /dev/null"
"2> /dev/null' % job.tmpdir,"
"ignore_status=True)")
control_new.append("\t system('rm -rf ' + job.tmpdir)")
control_new.append(
'b0 = job.barrier("%s", "sc_bar", %d, port=%d)'
% (jobid, sc_bar_timeout, sc_bar_port))
control_new.append(
'b0.rendezvous_servers("PARALLEL_MASTER", "%s")'
% jobid)
elif instance == 1:
# Wait at the server barrier to wait for instance=0
# process to complete setup
b0 = barrier.barrier("PARALLEL_MASTER", "sc_bar", sc_bar_timeout,
port=sc_bar_port)
b0.rendezvous_servers("PARALLEL_MASTER", jobid)
if(num_jobs > 2):
b1 = barrier.barrier(jobid, "s_bar", s_bar_timeout,
port=s_bar_port)
b1.rendezvous(rendvstr)
else:
# For the rest of the clients
b2 = barrier.barrier(jobid, "s_bar", s_bar_timeout, port=s_bar_port)
b2.rendezvous(rendvstr)
# Client side barrier for all the tests to start at the same time
control_new.append('b1 = job.barrier("%s", "c_bar", %d, port=%d)'
% (jobid, c_bar_timeout, c_bar_port))
control_new.append("b1.rendezvous(%s)" % rendvstr)
# Stick in the rest of the control file
control_new.append(control)
return "\n".join(control_new)
def barrier(self, *args, **kwds):
"""Create a barrier object"""
return barrier.barrier(*args, **kwds)
def barrier(self, *args, **kwds):
"""Create a barrier object"""
return barrier.barrier(*args, **kwds)
def get_sync_control_file(control,
host_name,
host_num,
instance,
num_jobs,
port_base=63100):
"""
This function is used when there is a need to run more than one
job simultaneously starting exactly at the same time. It basically returns
a modified control file (containing the synchronization code prepended)
whenever it is ready to run the control file. The synchronization
is done using barriers to make sure that the jobs start at the same time.
Here is how the synchronization is done to make sure that the tests
start at exactly the same time on the client.
sc_bar is a server barrier and s_bar, c_bar are the normal barriers
Job1 Job2 ...... JobN
Server: | sc_bar
Server: | s_bar ...... s_bar
Server: | at.run() at.run() ...... at.run()
----------|------------------------------------------------------
Client | sc_bar
Client | c_bar c_bar ...... c_bar
Client | <run test> <run test> ...... <run test>
@param control: The control file which to which the above synchronization
code will be prepended.
@param host_name: The host name on which the job is going to run.
@param host_num: (non negative) A number to identify the machine so that
we have different sets of s_bar_ports for each of the machines.
@param instance: The number of the job
@param num_jobs: Total number of jobs that are going to run in parallel
with this job starting at the same time.
@param port_base: Port number that is used to derive the actual barrier
ports.
@returns The modified control file.
"""
sc_bar_port = port_base
c_bar_port = port_base
if host_num < 0:
print "Please provide a non negative number for the host"
return None
s_bar_port = port_base + 1 + host_num # The set of s_bar_ports are
# the same for a given machine
sc_bar_timeout = 180
s_bar_timeout = c_bar_timeout = 120
# The barrier code snippet is prepended into the conrol file
# dynamically before at.run() is called finally.
control_new = []
# jobid is the unique name used to identify the processes
# trying to reach the barriers
jobid = "%s#%d" % (host_name, instance)
rendv = []
# rendvstr is a temp holder for the rendezvous list of the processes
for n in range(num_jobs):
rendv.append("'%s#%d'" % (host_name, n))
rendvstr = ",".join(rendv)
if instance == 0:
# Do the setup and wait at the server barrier
# Clean up the tmp and the control dirs for the first instance
control_new.append('if os.path.exists(job.tmpdir):')
control_new.append("\t system('umount -f %s > /dev/null"
"2> /dev/null' % job.tmpdir,"
"ignore_status=True)")
control_new.append("\t system('rm -rf ' + job.tmpdir)")
control_new.append('b0 = job.barrier("%s", "sc_bar", %d, port=%d)' %
(jobid, sc_bar_timeout, sc_bar_port))
control_new.append('b0.rendezvous_servers("PARALLEL_MASTER", "%s")' %
jobid)
elif instance == 1:
# Wait at the server barrier to wait for instance=0
# process to complete setup
b0 = barrier.barrier("PARALLEL_MASTER",
"sc_bar",
sc_bar_timeout,
port=sc_bar_port)
b0.rendezvous_servers("PARALLEL_MASTER", jobid)
if (num_jobs > 2):
b1 = barrier.barrier(jobid,
"s_bar",
s_bar_timeout,
port=s_bar_port)
b1.rendezvous(rendvstr)
else:
# For the rest of the clients
b2 = barrier.barrier(jobid, "s_bar", s_bar_timeout, port=s_bar_port)
b2.rendezvous(rendvstr)
# Client side barrier for all the tests to start at the same time
control_new.append('b1 = job.barrier("%s", "c_bar", %d, port=%d)' %
(jobid, c_bar_timeout, c_bar_port))
control_new.append("b1.rendezvous(%s)" % rendvstr)
# Stick in the rest of the control file
control_new.append(control)
return "\n".join(control_new)
