def __init__(self, mock_api=False, trust=False, client_kwargs=None):
# instantiate api
api_class = MockAPI if mock_api else API
self.api = api_class(**(client_kwargs or {}))
self.logger = logging.getLogger('importer.api.eventful.consumer')
self.event_horizon = None
self.trust = trust
self.client_call_limit = conf.API_CALL_LIMIT if self.trust else conf.SAFE_API_CALL_LIMIT
self.venue_ids = set()
self.images_by_event_id = {}
self.images_by_venue_id = {}
self.venues_by_venue_id = {}
# a green pile for images w/ fairly high concurrency
# (webservers are ok with it)
self.event_image_pile = eventlet.GreenPile(15)
# a green pile for event details w/ lower concurrency
# (appservers are not too ok with it)
self.event_detail_pile = eventlet.GreenPile(10)
# a green pile for venue images w/ fairly high concurrency
# (webservers are ok with it)
self.venue_image_pile = eventlet.GreenPile(15)
# a green pile for venue details w/ lower concurrency
# (appservers are not too ok with it)
self.venue_detail_pile = eventlet.GreenPile(10)
def test_constructing_from_pool(self):
pool = eventlet.GreenPool(2)
pile1 = eventlet.GreenPile(pool)
pile2 = eventlet.GreenPile(pool)
def bunch_of_work(pile, unique):
for i in range(10):
pile.spawn(passthru, i + unique)
eventlet.spawn(bunch_of_work, pile1, 0)
eventlet.spawn(bunch_of_work, pile2, 100)
eventlet.sleep(0)
self.assertEqual(list(pile2), list(range(100, 110)))
self.assertEqual(list(pile1), list(range(10)))
def main():
query_message = ZeroAccessUtil.buildMessage()
SEPARATOR = '/'
if sys.platform == 'win32':
SEPARATOR = "\\"
message = ZeroAccessUtil.buildMessage()
print message.encode("hex")
nonQueryedNodes = mul.Queue(5000)
zeroaccess_bootstrap_seeds_path = "Data" + SEPARATOR + "zeroaccess_node.dat"
zeroaccess_nodes = ZeroAccessUtil.read_zeroaccess_data_from_file(
zeroaccess_bootstrap_seeds_path)
pile = eventlet.GreenPile()
for x in zeroaccess_nodes[:10]:
pile.spawn(query, x, message)
# note that the pile acts as a collection of return values from the functions
# if any exceptions are raised by the function they'll get raised here
key = [ord('2'), ord('p'), ord('t'), ord('f')]
for node, result in zip(zeroaccess_nodes[:10], pile):
if (result == ''):
print 'no response from ' + socket.inet_ntoa(
struct.pack('I', node.get_ip()))
continue
print 'received'
original_message = ZeroAccessUtil.xorMessage(result[0], key)
crc32, retL_command, b_flag, ip_count = struct.unpack(
'IIII', original_message[:16])
print socket.inet_ntoa(struct.pack(
'I', node.get_ip())) + ' --> ip count: ' + str(ip_count)
def _collect_status_data(self):
try:
now = datetime.datetime.now()
children = self.controller.children.values()
status_data = {
'active_children_count':
len([c for c in children if c.active]),
'killed_children_count':
len([c for c in children if not c.active]),
'configured_children_count': self.controller.num_processes,
'now': now.ctime(),
'pid': os.getpid(),
'uptime': format_timedelta(now - self.controller.started_at),
'started_at': self.controller.started_at.ctime(),
'config': self.controller.config
}
# fire up a few greenthreads to wait on children's responses
p = eventlet.GreenPile()
for child in self.controller.children.values():
p.spawn(self.collect_child_status, child)
status_data['children'] = dict([pid_cd for pid_cd in p])
# total concurrent connections
status_data['concurrent_requests'] = sum([
child.get('concurrent_requests', 0)
for child in status_data['children'].values()
])
finally:
# wipe out the waiter so that subsequent requests create new ones
self.status_waiter = None
return status_data
def app(environ, start_rsp):
pool = eventlet.GreenPool(100)
pile = eventlet.GreenPile(pool)
for url in environ['wsgi.input'].readlines():
pile.spawn(fetch_title, url)
titles = '\n'.join(pile)
start_rsp()
def test_wrapper_isolation(self):
pile = eventlet.GreenPile()
[pile.spawn(isolation_query, i) for i in range(10)]
for scope, scope_values in pile:
for val in scope_values:
self.assertEqual(scope, val)
def test_create_swift_account_fail(self):
self.ret_index = {}
self.pa = 0
def create_tenant(*args):
if self.pa == 0:
self.pa += 1
raise KSClientException('Fake msg')
else:
self.pa += 1
return FakeKSTenant('foo1')
def create_swift_user(*args):
pass
client = FakeKSClient()
self.stubs.Set(client.tenants, 'create', create_tenant)
self.stubs.Set(filler, 'create_swift_user', create_swift_user)
concurrency = int(
utils.get_config('concurrency',
'filler_keystone_client_concurrency'))
pile = eventlet.GreenPile(concurrency)
filler.create_swift_account(client, pile, 3, 1, self.ret_index)
self.assertEqual(len(self.ret_index.keys()), 2)
def test_a_buncha_stuff(self):
assert_ = self.assert_
class Dummy(object):
def foo(self, when, token=None):
assert_(token is not None)
time.sleep(random.random() / 200.0)
return token
def sender_loop(loopnum):
obj = tpool.Proxy(Dummy())
count = 100
for n in six.moves.range(count):
eventlet.sleep(random.random() / 200.0)
now = time.time()
token = loopnum * count + n
rv = obj.foo(now, token=token)
self.assertEqual(token, rv)
eventlet.sleep(random.random() / 200.0)
cnt = 10
pile = eventlet.GreenPile(cnt)
for i in six.moves.range(cnt):
pile.spawn(sender_loop, i)
results = list(pile)
self.assertEqual(len(results), cnt)
tpool.killall()
def main(self):
if self.args.bucket and self.args.bucket_folder:
self.conn = S3Connection()
self.bucket = self.conn.get_bucket(self.args.bucket)
rs = self.bucket.list(self.args.bucket_folder)
key_counter = 0
keys_to_delete = []
pile = eventlet.GreenPile(self.args.concurrency)
for key in rs:
keys_to_delete.append(key.name)
key_counter += 1
if key_counter % 1000 == 0:
pile.spawn(self.delete, keys_to_delete)
keys_to_delete = []
pile.spawn(self.delete, keys_to_delete) # Pick up stragglers
# Wait for all greenlets to finish
list(pile)
else:
print "Can't do it. Not enouch specified. You'll nuke the kernel."
def _transform_object(self, zobj):
tags = set()
metadata = dict()
zobj.metadata = Metadata()
d = zobj.transformed
pool = eventlet.GreenPool()
pile = eventlet.GreenPile(pool)
for tag in self.eligible_tags:
pile.spawn(match_pro, zobj, tag, self.logger, self.debug)
if zobj.metadata.empty():
for tag in self.common_tags:
# try:
# meta = Metadata()
# meta = tag.process(d, meta)
# if meta is not None:
# zobj.metadata.merge(meta)
# except Exception as e:
# if self.logger is not None:
# error = "%s: %s %s" % (tag.__class__.__name__,
# type(e).__name__, str(e))
# self.logger.debug(error)
# if self.debug:
# raise e
pile.spawn(match_pro, zobj, tag, self.logger, self.debug)
pool.waitall()
if zobj.metadata.empty() is False:
return zobj
def delete_container(self, dest_storage_cnx, dest_token, orig_containers,
dest_containers):
set1 = set((x['name']) for x in orig_containers)
set2 = set((x['name']) for x in dest_containers)
delete_diff = set2 - set1
pool = eventlet.GreenPool(size=self.concurrency)
pile = eventlet.GreenPile(pool)
for container in delete_diff:
try:
dest_container_stats, dest_objects = swiftclient.get_container(
None,
dest_token,
container,
http_conn=dest_storage_cnx,
full_listing=True,
)
except (swiftclient.client.ClientException), e:
logging.info("error getting container: %s, %s" %
(container, e.http_reason))
continue
for obj in dest_objects:
logging.info("deleting obj: %s ts:%s", obj['name'],
obj['last_modified'])
pile.spawn(self.delete_object, dest_storage_cnx, dest_token,
container, obj['name'])
pool.waitall()
logging.info("deleting container: %s", container)
pile.spawn(swiftclient.delete_container,
'',
dest_token,
container,
http_conn=dest_storage_cnx)
def test_start_many_slow_connections(self):
""" Test starting many slow connections """
pile = eventlet.GreenPile()
[pile.spawn(self.get_connection) for _ in xrange(self.NUM_ITER)]
for conn in pile:
self.assertIsInstance(conn, RexProEventletConnection)
def spawn_order_check(self, concurrency):
# checks that piles are strictly ordered
p = eventlet.GreenPile(concurrency)
def makework(count, unique):
for i in six.moves.range(count):
token = (unique, i)
p.spawn(pressure, token)
iters = 1000
eventlet.spawn(makework, iters, 1)
eventlet.spawn(makework, iters, 2)
eventlet.spawn(makework, iters, 3)
p.spawn(pressure, (0, 0))
latest = [-1] * 4
received = 0
it = iter(p)
while True:
try:
i = six.next(it)
except StressException as exc:
i = exc.args[0]
except StopIteration:
break
received += 1
if received % 5 == 0:
eventlet.sleep(0.0001)
unique, order = i
assert latest[unique] < order
latest[unique] = order
for l in latest[1:]:
self.assertEqual(l, iters - 1)
def geturls():
urls = ['www.google.com', 'www.yandex.ru', 'www.python.org']
pile = eventlet.GreenPile()
for url in urls:
pile.spawn(geturl, url)
for url, rsp in zip(urls, pile):
print('%s: %s' % (url, rsp))
def test_unique_connections(self):
pool = self.get_pool()
pile = eventlet.GreenPile()
for i in xrange(self.NUM_ITER):
pile.spawn(self.spawn_slow_network_and_query_slow_response, pool,
self.SLOW_NETWORK_QUERY, 1, {
'value': i,
i: 'value'
})
def setup_siege_urls(hostnames):
"""Launches a coroutine to write siege urls based on user input."""
print ' Configuring urls... ',
pile = eventlet.GreenPile(pool)
for h in hostnames:
pile.spawn(_setup_siege_urls, h)
responses = list(pile)
print 'Done!'
return responses
def run_cmd_on_nodes(node_list, cmd):
pile = eventlet.GreenPile()
results = {}
for node in node_list:
pile.spawn(run_cmd_on_node, node, cmd)
for node, res in zip(node_list, pile):
results[node] = res
logging.disable(logging.NOTSET)
return results
def check_hosts(self, hosts):
hosts_statics = {}
pile = eventlet.GreenPile()
for host in hosts:
pile.spawn(self._check_host, host)
for host, host_state in zip(hosts, pile):
hosts_statics[host] = host_state
return hosts_statics
def run_doctor_on_nodes(node_list, check_cmd):
pile = eventlet.GreenPile()
result = []
for node in node_list:
pile.spawn(run_doctor_cmd_on_node, node['role'], node['name'],
check_cmd)
for node, res in zip(node_list, pile):
result.append(res)
logging.disable(logging.NOTSET)
return result
def test_contention(self):
from tests import tpool_test
prox = tpool.Proxy(tpool_test)
pile = eventlet.GreenPile(4)
pile.spawn(lambda: self.assertEqual(prox.one, 1))
pile.spawn(lambda: self.assertEqual(prox.two, 2))
pile.spawn(lambda: self.assertEqual(prox.three, 3))
results = list(pile)
self.assertEqual(len(results), 3)
def test_nested_values(self):
pile = eventlet.GreenPile()
[pile.spawn(nested_wrappers, chr(i)) for i in range(97, 103)]
for scope, scope_v in pile:
self.assertEqual(scope, scope_v.name)
for inner_scope, scope_values in scope_v['nested_values']:
for val in scope_values:
self.assertEqual(inner_scope, val)
scope_v.delete()
def setup_cannons(hostnames):
"""Launches a coroutine to configure each host and waits for them to
complete before compiling a list of responses
"""
print ' Loading cannons... ',
pile = eventlet.GreenPile(pool)
for h in hostnames:
pile.spawn(_setup_a_cannon, h)
responses = list(pile)
print 'Done!'
return responses
def mbtiles_grid_tiles(self):
"""
Get grid tiles and upload.
"""
tile_count = self.mbtiles.execute(
'select count(zoom_level) from grids;').fetchone()[0]
if not tile_count > 0:
return False
# Progress bar
widgets = [
'- Uploading %s grid tiles: ' % (tile_count),
progressbar.Percentage(), ' ',
progressbar.Bar(), ' ',
progressbar.ETA()
]
progress = progressbar.ProgressBar(widgets=widgets,
maxval=tile_count).start()
completed = 0
# Create eventlet pile
pile = eventlet.GreenPile(self.args.concurrency)
# Get tiles
tiles = self.mbtiles.execute(
'select zoom_level, tile_column, tile_row, grid from grids;')
t = tiles.fetchone()
while t:
key = '%s/%s/%s/%s.grid.json' % (self.tileset, t[0], t[1], t[2])
# Get actual json data
grid_data = self.mbtiles.execute(
'select key_name, key_json FROM grid_data WHERE zoom_level = %s and tile_column = %s and tile_row = %s;'
% (t[0], t[1], t[2])).fetchall()
grid_data_parse = {}
for d in grid_data:
grid_data_parse[d[0]] = json.loads(d[1])
# Put together
grid = json.loads(zlib.decompress(t[3]).decode('utf-8'))
grid['data'] = grid_data_parse
# Upload data
(grid, mime_type) = self.jsonp(grid)
pile.spawn(self.send_file, key, grid, mime_type=mime_type)
# Get next and update
t = tiles.fetchone()
completed = completed + 1
progress.update(completed)
# Wait for pile and stop progress bar
list(pile)
progress.finish()
def get_historical_data(symbol):
if symbol:
companies = ListedCompany.objects.filter(symbol=symbol)
else:
companies = ListedCompany.objects.all()[:10]
print "loading" + str(companies)
pool = eventlet.GreenPool(5)
pile = eventlet.GreenPile(pool)
for company in companies:
print company
pile.spawn(get_company_data, company)
def run_doctor_on_nodes(node_list, check_cmd):
pile = eventlet.GreenPile()
result = []
for node in node_list:
LOG.info('%s%s Push check cmd to %-13s (%-10s) %s%s' %
('<', '=' * 2, node['name'], node['role'], '=' * 2, '>'))
pile.spawn(run_doctor_cmd_on_node, node['role'], node['name'],
check_cmd)
for node, res in zip(node_list, pile):
result.append(res)
logging.disable(logging.NOTSET)
return result
def test_pile_spawn_times_out(self):
p = eventlet.GreenPile(4)
for i in range(4):
p.spawn(passthru, i)
# now it should be full and this should time out
eventlet.Timeout(0)
self.assertRaises(eventlet.Timeout, p.spawn, passthru, "time out")
# verify that the spawn breakage didn't interrupt the sequence
# and terminates properly
for i in range(4, 10):
p.spawn(passthru, i)
self.assertEqual(list(p), list(range(10)))
def slam_host(cannon_hosts, target):
"""Coordinates `cannon_hosts` to use the specified siege coordates on
`target` and report back the performance.
"""
pile = eventlet.GreenPile(pool)
for h in cannon_hosts:
pile.spawn(fire_cannon, h, target)
responses = list(pile)
try:
report = parse_responses(responses)
except UnparsableData, e:
return "Unable to parse data properly: %s" % e
def test_many_network_calls(self):
""" Test known responses on a network that should be slow, we should get them all asynchronously """
pile = eventlet.GreenPile()
for i in xrange(self.NUM_ITER):
pile.spawn(spawn_slow_network_and_query_slow_response, self,
self.SLOW_NETWORK_QUERY, 1, {
'value': i,
i: 'value'
})
for result in pile:
print_(result)
self.assertIsInstance(result, dict)
def app(environ, start_response):
if environ["REQUEST_METHOD"] != "POST":
start_response("403 Forbidden", [])
return []
pile = eventlet.GreenPile(pool)
for line in environ["wsgi.input"].readlines():
print(line)
url = line.strip()
if url:
pile.spawn(fetch, url)
titles = "\n".join(pile)
start_response("200 OK", [{"Content-Type", "text/plain"}])
return [titles]
def app(environ, start_response):
if environ['REQUEST_METHOD'] != 'POST':
start_response('403 Forbidden', [])
return []
# the pile collects the result of a concurrent operation -- in this case,
# the collection of feed titles
pile = eventlet.GreenPile(pool)
for line in environ['wsgi.input'].readlines():
url = line.strip()
if url:
pile.spawn(fetch_title, url)
# since the pile is an iterator over the results,
# you can use it in all sorts of great Pythonic ways
titles = '\n'.join(pile)
start_response('200 OK', [('Content-type', 'text/plain')])
return [titles]
