def test_remove_from_labels(self):
marktime.start('test run')
marktime.stop('test run')
self.assertIn('test run', marktime.labels)
marktime.stop('test run', remove_from_labels=True)
self.assertNotIn('test run', marktime.labels)
def test_stop_twice(self, mock_time):
mock_time.return_value = 123
marktime.start('test run')
self.assertEquals(marktime.stop('test run', at=124).seconds, 1)
self.assertEquals(marktime.stop('test run', at=125).seconds, 1)
self.assertEquals(marktime.stop('test run', at=183,
stop_once=False).minutes, 1)
def test_duration(self):
start_time = 1370451294
diff_time = round(random.random() * 100)
stop_time = start_time + diff_time
marktime.start('test run', at=start_time)
marktime.stop('test run', at=stop_time)
self.assertEquals(marktime.duration('test run').seconds, diff_time)
def claim_delete(queues, stats, test_duration, ttl, grace, limit):
"""Consumer Worker
The Consumer Worker continuously claims and deletes messages
for the specified duration. The time taken for each claim and
delete is recorded for calculating throughput and latency.
"""
end = time.time() + test_duration
claim_total_elapsed = 0
delete_total_elapsed = 0
total_failed_requests = 0
claim_total_requests = 0
delete_total_requests = 0
while time.time() < end:
# NOTE(kgriffs): Distribute requests across all queues evenly.
queue = random.choice(queues)
try:
marktime.start('claim_message')
claim = queue.claim(ttl=ttl, grace=grace, limit=limit)
claim_total_elapsed += marktime.stop('claim_message').seconds
claim_total_requests += 1
except errors.TransportError as ex:
sys.stderr.write("Could not claim messages : {0}\n".format(ex))
total_failed_requests += 1
else:
for msg in claim:
try:
marktime.start('delete_message')
msg.delete()
elapsed = marktime.stop('delete_message').seconds
delete_total_elapsed += elapsed
delete_total_requests += 1
except errors.TransportError as ex:
msg = "Could not delete messages: {0}\n".format(ex)
sys.stderr.write(msg)
total_failed_requests += 1
total_requests = (claim_total_requests +
delete_total_requests +
total_failed_requests)
stats.put({
'total_requests': total_requests,
'claim_total_requests': claim_total_requests,
'delete_total_requests': delete_total_requests,
'claim_total_elapsed': claim_total_elapsed,
'delete_total_elapsed': delete_total_elapsed,
})
def claim_delete(queues, stats, test_duration, ttl, grace, limit):
"""Consumer Worker
The Consumer Worker continuously claims and deletes messages
for the specified duration. The time taken for each claim and
delete is recorded for calculating throughput and latency.
"""
end = time.time() + test_duration
claim_total_elapsed = 0
delete_total_elapsed = 0
total_failed_requests = 0
claim_total_requests = 0
delete_total_requests = 0
while time.time() < end:
# NOTE(kgriffs): Distribute requests across all queues evenly.
queue = random.choice(queues)
try:
marktime.start("claim_message")
claim = queue.claim(ttl=ttl, grace=grace, limit=limit)
claim_total_elapsed += marktime.stop("claim_message").seconds
claim_total_requests += 1
except errors.TransportError as ex:
sys.stderr.write("Could not claim messages : {0}\n".format(ex))
total_failed_requests += 1
else:
for msg in claim:
try:
marktime.start("delete_message")
msg.delete()
elapsed = marktime.stop("delete_message").seconds
delete_total_elapsed += elapsed
delete_total_requests += 1
except errors.TransportError as ex:
msg = "Could not delete messages: {0}\n".format(ex)
sys.stderr.write(msg)
total_failed_requests += 1
total_requests = claim_total_requests + delete_total_requests + total_failed_requests
stats.put(
{
"total_requests": total_requests,
"claim_total_requests": claim_total_requests,
"delete_total_requests": delete_total_requests,
"claim_total_elapsed": claim_total_elapsed,
"delete_total_elapsed": delete_total_elapsed,
}
)
def claim_delete(stats, test_duration, ttl, grace, limit):
"""Consumer Worker
The Consumer Worker continuously claims and deletes messages
for the specified duration. The time taken for each claim and
delete is recorded for calculating throughput and latency.
"""
cli = client.Client(conf.server_url)
queue = cli.queue(conf.queue_prefix + '1')
end = time.time() + test_duration
total_elapsed = 0
total_requests = 0
claim_total_requests = 0
delete_total_requests = 0
while time.time() < end:
marktime.start('claim_message')
try:
claim = queue.claim(ttl=ttl, grace=grace, limit=limit)
except TransportError as ex:
sys.stderr.write("Could not claim messages : {0}\n".format(ex))
else:
total_elapsed += marktime.stop('claim_message').seconds
claim_total_requests += 1
try:
marktime.start('delete_message')
for msg in claim:
# TODO(TheSriram): Simulate actual work before deletion
msg.delete()
total_elapsed += marktime.stop('delete_message').seconds
delete_total_requests += 1
except TransportError as ex:
sys.stderr.write("Could not delete messages: {0}\n".format(ex))
finally:
total_requests += 1
finally:
total_requests += 1
stats.put({'total_requests': total_requests,
'claim_total_requests': claim_total_requests,
'delete_total_requests': delete_total_requests,
'total_elapsed': total_elapsed})
def claim_delete(stats, test_duration, ttl, grace, limit):
"""Consumer Worker
The Consumer Worker continuously claims and deletes messages
for the specified duration. The time taken for each claim and
delete is recorded for calculating throughput and latency.
"""
cli = client.Client(URL)
queue = cli.queue(QUEUE_PREFIX + '1')
end = time.time() + test_duration
total_elapsed = 0
total_requests = 0
claim_total_requests = 0
delete_total_requests = 0
while time.time() < end:
marktime.start('claim_message')
try:
claim = queue.claim(ttl=ttl, grace=grace, limit=limit)
except TransportError as ex:
print("Could not claim messages : {0}".format(ex))
else:
total_elapsed += marktime.stop('claim_message').seconds
total_requests += 1
claim_total_requests += 1
try:
marktime.start('delete_message')
for msg in claim:
# TODO(TheSriram): Simulate actual work before deletion
msg.delete()
total_elapsed += marktime.stop('delete_message').seconds
delete_total_requests += 1
total_requests += 1
stats.put({
'total_requests': total_requests,
'claim_total_requests': claim_total_requests,
'delete_total_requests': delete_total_requests,
'total_elapsed': total_elapsed
})
except TransportError as ex:
print("Could not claim and delete : {0}".format(ex))
def producer(stats, test_duration):
"""Producer Worker
The Producer Worker continuously post messages for
the specified duration. The time taken for each post
is recorded for calculating throughput and latency.
"""
cli = client.Client(URL)
queue = cli.queue(QUEUE_PREFIX + '1')
total_requests = 0
total_elapsed = 0
end = time.time() + test_duration
while time.time() < end:
marktime.start('post message')
# TODO(TheSriram): Track/report errors
try:
queue.post(choose_message())
except TransportError as ex:
print("Could not post a message : {0}".format(ex))
else:
total_elapsed += marktime.stop('post message').seconds
total_requests += 1
stats.put({
'total_requests': total_requests,
'total_elapsed': total_elapsed
})
def producer(stats, test_duration):
"""Producer Worker
The Producer Worker continuously post messages for
the specified duration. The time taken for each post
is recorded for calculating throughput and latency.
"""
cli = client.Client(URL)
queue = cli.queue(QUEUE_PREFIX + '1')
total_requests = 0
total_elapsed = 0
end = time.time() + test_duration
while time.time() < end:
marktime.start('post message')
# TODO(TheSriram): Track/report errors
try:
queue.post(choose_message())
except TransportError as ex:
print("Could not post a message : {0}".format(ex))
else:
total_elapsed += marktime.stop('post message').seconds
total_requests += 1
stats.put({
'total_requests': total_requests,
'total_elapsed': total_elapsed
})
def task_done(task_name):
"""
Stop task in marktime and return timedelta object for duration
:param task_name:
:return:
"""
return timedelta(seconds=marktime.stop(task_name).value)
def test_duration_data_in_dumps(self, mock_time):
mock_time.return_value = 123
marktime.start('test run')
time_diff = marktime.stop('test run', at=124,
remove_from_labels=False).seconds
self.assertEquals(marktime.labels['test run']['duration'], time_diff)
def producer(queues, message_pool, stats, test_duration):
"""Producer Worker
The Producer Worker continuously post messages for
the specified duration. The time taken for each post
is recorded for calculating throughput and latency.
"""
total_requests = 0
successful_requests = 0
total_elapsed = 0
end = time.time() + test_duration
while time.time() < end:
queue = random.choice(queues)
try:
marktime.start('post_message')
queue.post(choose_message(message_pool))
total_elapsed += marktime.stop('post_message').seconds
successful_requests += 1
except errors.TransportError as ex:
sys.stderr.write("Could not post a message : {0}\n".format(ex))
total_requests += 1
stats.put({
'successful_requests': successful_requests,
'total_requests': total_requests,
'total_elapsed': total_elapsed
})
def producer(queues, message_pool, stats, test_duration):
"""Producer Worker
The Producer Worker continuously post messages for
the specified duration. The time taken for each post
is recorded for calculating throughput and latency.
"""
total_requests = 0
successful_requests = 0
total_elapsed = 0
end = time.time() + test_duration
while time.time() < end:
queue = random.choice(queues)
try:
marktime.start('post_message')
queue.post(choose_message(message_pool))
total_elapsed += marktime.stop('post_message').seconds
successful_requests += 1
except errors.TransportError as ex:
sys.stderr.write("Could not post a message : {0}\n".format(ex))
total_requests += 1
stats.put({
'successful_requests': successful_requests,
'total_requests': total_requests,
'total_elapsed': total_elapsed
})
def test_real_times(self, mock_time):
start_time = 1370451294.106749
diff_time = random.random() * 100
stop_time = start_time + diff_time
mock_time.return_value = start_time
marktime.start('test run')
mock_time.return_value = stop_time
self.assertEquals(round(marktime.stop('test run').seconds, 4),
round(diff_time, 4))
def test_severals_markers(self, mock_time):
start_time = 123
markers_count = 10
mock_time.return_value = start_time
for i in range(1, markers_count):
marktime.start('test run %d' % i)
for i in range(1, markers_count):
time_diff = marktime.stop('test run %d' % i,
at=(start_time + i)).seconds
self.assertEquals(time_diff, i)
def observer(queues, stats, test_duration, limit):
"""Observer Worker
The observer lists messages without claiming them.
"""
end = time.time() + test_duration
total_elapsed = 0
total_succeeded = 0
total_failed = 0
queues = [{'q': q, 'm': None} for q in queues]
while time.time() < end:
# NOTE(kgriffs): Distribute requests across all queues evenly.
queue = random.choice(queues)
try:
marktime.start('list_messages')
cursor = queue['q'].messages(limit=limit,
marker=queue['m'],
include_claimed=True)
total_elapsed += marktime.stop('list_messages').seconds
total_succeeded += 1
messages = list(cursor)
if messages:
# TODO(kgriffs): Figure out a less hacky way to do this
# while preserving the ability to measure elapsed time
# per request.
queue['m'] = _extract_marker(cursor._links)
except errors.TransportError as ex:
sys.stderr.write("Could not list messages : {0}\n".format(ex))
total_failed += 1
total_requests = total_succeeded + total_failed
stats.put({
'total_requests': total_requests,
'total_succeeded': total_succeeded,
'total_elapsed': total_elapsed,
})
def observer(queues, stats, test_duration, limit):
"""Observer Worker
The observer lists messages without claiming them.
"""
end = time.time() + test_duration
total_elapsed = 0
total_succeeded = 0
total_failed = 0
queues = [{'q': q, 'm': None} for q in queues]
while time.time() < end:
# NOTE(kgriffs): Distribute requests across all queues evenly.
queue = random.choice(queues)
try:
marktime.start('list_messages')
cursor = queue['q'].messages(limit=limit, marker=queue['m'],
include_claimed=True)
total_elapsed += marktime.stop('list_messages').seconds
total_succeeded += 1
messages = list(cursor)
if messages:
# TODO(kgriffs): Figure out a less hacky way to do this
# while preserving the ability to measure elapsed time
# per request.
queue['m'] = _extract_marker(cursor._links)
except errors.TransportError as ex:
sys.stderr.write("Could not list messages : {0}\n".format(ex))
total_failed += 1
total_requests = total_succeeded + total_failed
stats.put({
'total_requests': total_requests,
'total_succeeded': total_succeeded,
'total_elapsed': total_elapsed,
})
action_params[thing] = getattr(agent, thing)
update_dict = action(agent.state, env.state, action_params)
update_dict.apply(agent.state)
env.register_action(n, action, action_params)
env._tick()
# Gather rewards
temp_fitnesses = []
for s in range(len(solvers)):
temp_fitnesses.append([])
for agent in agents:
solver_index = agent_types.index(agent.__class__.__name__)
agent_reward = agent.state[agent_reward_vars[agent.name]]
temp_fitnesses[solver_index].append(agent_reward)
for f, thing in enumerate(temp_fitnesses):
avg = np.mean(thing)
all_fitnesses[f][k] = avg
for g, thing in enumerate(all_fitnesses):
solvers[g].tell(thing)
duration = mt.stop("run")
if args.verbose:
print "Run %i complete in %f seconds" % (i, duration.seconds)
use multiprocessing.pool.ThreadPool instead
'''
import marktime # stopwatch
from multiprocessing.pool import ThreadPool
def foo(bar, baz):
print('hello {0}'.format(bar))
return 'foo' + baz
def foo2(bar, baz):
print('hello2 ' + bar)
return 'foo2 ' + baz
def main():
pool = ThreadPool(processes=1)
async_result = pool.apply_async(foo, ('world', 'foo')) # tuple of args for foo
async_result2 = pool.apply_async(foo2, ('world', 'foo2'))
# do some other stuff in the main process
return_val = async_result.get() # get the return value from your function.
return_val2 = async_result2.get()
print(return_val)
print(return_val2)
if __name__ == "__main__":
marktime.start('task')
main()
marktime.stop('task')
print(marktime.duration('task').msecs)
def test_float_diffs(self, mock_time):
mock_time.return_value = 123
marktime.start('test run')
self.assertEquals(marktime.stop('test run', at=124.5).seconds, 1.5)
def test_start_stop_at(self):
marktime.start('test run', at=123)
self.assertEquals(marktime.stop('test run', at=124).msecs, 1000)
def test_start_stop(self, mock_time):
mock_time.return_value = 123
marktime.start('test run')
self.assertEquals(marktime.stop('test run', at=124).seconds, 1)
def test_stop_label_not_exists(self):
self.assertIsNone(marktime.stop('not existance label'))
