def _full_updatemacmap(configmanager): global vintage global _macmap global _nodesbymac global _switchportmap global _macsbyswitch global switchbackoff start = util.monotonic_time() with mapupdating: vintage = util.monotonic_time() # Clear all existing entries _macmap = {} _nodesbymac = {} _switchportmap = {} _macsbyswitch = {} if configmanager.tenant is not None: raise exc.ForbiddenRequest( 'Network topology not available to tenants') # here's a list of switches... need to add nodes that are switches nodelocations = configmanager.get_node_attributes( configmanager.list_nodes(), ('net*.switch', 'net*.switchport')) switches = set([]) for node in nodelocations: cfg = nodelocations[node] for attr in cfg: if not attr.endswith('.switch') or 'value' not in cfg[attr]: continue curswitch = cfg[attr].get('value', None) if not curswitch: continue switches.add(curswitch) switchportattr = attr + 'port' if switchportattr in cfg: portname = cfg[switchportattr].get('value', '') if not portname: continue if curswitch not in _switchportmap: _switchportmap[curswitch] = {} if portname in _switchportmap[curswitch]: log.log({ 'error': 'Duplicate switch topology config ' 'for {0} and {1}'.format( node, _switchportmap[curswitch][portname]) }) _switchportmap[curswitch][portname] = None else: _switchportmap[curswitch][portname] = node switchauth = get_switchcreds(configmanager, switches) pool = GreenPool(64) for ans in pool.imap(_map_switch, switchauth): vintage = util.monotonic_time() yield ans endtime = util.monotonic_time() duration = endtime - start duration = duration * 15 # wait 15 times as long as it takes to walk # avoid spending a large portion of the time hitting switches with snmp # requests if duration > switchbackoff: switchbackoff = duration
def imap(self, func, *args): reqid = request_id() def impl(*args): set_request_id(reqid) return func(*args) return GreenPool.imap(self, impl, *args)
def update_macmap(configmanager): """Interrogate switches to build/update mac table Begin a rebuild process. This process is a generator that will yield as each switch interrogation completes, allowing a caller to recheck the cache as results become possible, rather than having to wait for the process to complete to interrogate. """ global _macmap global _nodesbymac global _switchportmap # Clear all existing entries _macmap = {} _nodesbymac = {} _switchportmap = {} if configmanager.tenant is not None: raise exc.ForbiddenRequest('Network topology not available to tenants') nodelocations = configmanager.get_node_attributes( configmanager.list_nodes(), ('hardwaremanagement.switch', 'hardwaremanagement.switchport')) switches = set([]) for node in nodelocations: cfg = nodelocations[node] if 'hardwaremanagement.switch' in cfg: curswitch = cfg['hardwaremanagement.switch']['value'] switches.add(curswitch) if 'hardwaremanagement.switchport' in cfg: portname = cfg['hardwaremanagement.switchport']['value'] if curswitch not in _switchportmap: _switchportmap[curswitch] = {} if portname in _switchportmap[curswitch]: log.log({ 'warning': 'Duplicate switch topology config for ' '{0} and {1}'.format( node, _switchportmap[curswitch][portname]) }) _switchportmap[curswitch][portname] = node switchcfg = configmanager.get_node_attributes( switches, ('secret.hardwaremanagementuser', 'secret.hardwaremanagementpassword'), decrypt=True) switchauth = [] for switch in switches: password = '******' user = None if (switch in switchcfg and 'secret.hardwaremanagementpassword' in switchcfg[switch]): password = switchcfg[switch]['secret.hardwaremanagementpassword'][ 'value'] if 'secret.hardwaremanagementuser' in switchcfg[switch]: user = switchcfg[switch]['secret.hardwaremanagementuser'][ 'value'] switchauth.append((switch, password, user)) pool = GreenPool() for res in pool.imap(_map_switch, switchauth): yield res print(repr(_macmap))
def update_macmap(configmanager): """Interrogate switches to build/update mac table Begin a rebuild process. This process is a generator that will yield as each switch interrogation completes, allowing a caller to recheck the cache as results become possible, rather than having to wait for the process to complete to interrogate. """ global _macmap global _nodesbymac global _switchportmap # Clear all existing entries _macmap = {} _nodesbymac = {} _switchportmap = {} if configmanager.tenant is not None: raise exc.ForbiddenRequest('Network topology not available to tenants') nodelocations = configmanager.get_node_attributes( configmanager.list_nodes(), ('hardwaremanagement.switch', 'hardwaremanagement.switchport')) switches = set([]) for node in nodelocations: cfg = nodelocations[node] if 'hardwaremanagement.switch' in cfg: curswitch = cfg['hardwaremanagement.switch']['value'] switches.add(curswitch) if 'hardwaremanagement.switchport' in cfg: portname = cfg['hardwaremanagement.switchport']['value'] if curswitch not in _switchportmap: _switchportmap[curswitch] = {} if portname in _switchportmap[curswitch]: log.log({'warning': 'Duplicate switch topology config for ' '{0} and {1}'.format(node, _switchportmap[ curswitch][ portname])}) _switchportmap[curswitch][portname] = node switchcfg = configmanager.get_node_attributes( switches, ('secret.hardwaremanagementuser', 'secret.hardwaremanagementpassword'), decrypt=True) switchauth = [] for switch in switches: password = '******' user = None if (switch in switchcfg and 'secret.hardwaremanagementpassword' in switchcfg[switch]): password = switchcfg[switch]['secret.hardwaremanagementpassword'][ 'value'] if 'secret.hardwaremanagementuser' in switchcfg[switch]: user = switchcfg[switch]['secret.hardwaremanagementuser'][ 'value'] switchauth.append((switch, password, user)) pool = GreenPool() for res in pool.imap(_map_switch, switchauth): yield res print(repr(_macmap))
def _update_neighbors_backend(configmanager, force): global _neighdata global _neighbypeerid vintage = _neighdata.get('!!vintage', 0) now = util.monotonic_time() if vintage > (now - 60) and not force: return _neighdata = {'!!vintage': now} _neighbypeerid = {'!!vintage': now} switches = netutil.list_switches(configmanager) switchcreds = netutil.get_switchcreds(configmanager, switches) switchcreds = [ x + (force,) for x in switchcreds] pool = GreenPool(64) for ans in pool.imap(_extract_neighbor_data, switchcreds): yield ans
def imap(requests, prefetch=True, size=2): """Concurrently converts a generator object of Requests to a generator of Responses. :param requests: a generator of Request objects. :param prefetch: If False, the content will not be downloaded immediately. :param size: Specifies the number of requests to make at a time. default is 2 """ def send(r): r.send(prefetch) return r.response pool = GreenPool(size) for r in pool.imap(send, requests): yield r pool.waitall()
def build_main(commits, config): logger = logging.getLogger(__name__ + '.build_main') logger.info('triggered with %d commits', len(commits)) logger.debug('commits = %r', commits) token = get_token(config) pool = GreenPool() commit_map = dict(commits) commit_ids = [commit_id for commit_id, _ in commits] results = pool.imap( functools.partial(download_archive, token=token, config=config), commit_ids ) env = make_virtualenv(config) save_dir = config['SAVE_DIRECTORY'] complete_hook = config.get('COMPLETE_HOOK') for commit, filename in results: working_dir = extract(filename, save_dir) try: build = build_sphinx(working_dir, env) except Exception: permalink = commit_map[commit] if not config.get('RECREATE_VIRTUALENV'): env = make_virtualenv(config, recreate=True) try: build = build_sphinx(working_dir, env) except Exception: if callable(complete_hook): complete_hook(commit, permalink, sys.exc_info()) continue if callable(complete_hook): complete_hook(commit, permalink, sys.exc_info()) continue result_dir = os.path.join(save_dir, commit) shutil.move(build, result_dir) logger.info('build complete: %s' % result_dir) if callable(complete_hook): complete_hook(commit, commit_map[commit], None) shutil.rmtree(working_dir) logger.info('working directory %s has removed' % working_dir) with open_head_file('w', config=config) as f: f.write(commit) logger.info('new head: %s', commits[0])
def run_stock_parser(): stock_symbols = [] with open('symbols.txt', 'r') as symfile: for n, line in enumerate(symfile): sym = line.strip() if sym: stock_symbols.append(sym) ncpu = cpu_count() pool = GreenPool(ncpu * 4) stock_prices = [] for symbol, price in pool.imap(read_stock_url, stock_symbols): stock_prices.append((symbol, price)) with open('stock_prices.csv', 'w') as outfile: outfile.write('Stock,Price\n') for symbol, price in stock_prices: outfile.write('%s,%s\n' % (symbol, price))
def build_main(commits, config): logger = logging.getLogger(__name__ + '.build_main') logger.info('triggered with %d commits', len(commits)) logger.debug('commits = %r', commits) token = get_token(config) pool = GreenPool() commit_map = dict(commits) commit_ids = [commit_id for commit_id, _ in commits] results = pool.imap( functools.partial(download_archive, token=token, config=config), commit_ids) env = make_virtualenv(config) save_dir = config['SAVE_DIRECTORY'] complete_hook = config.get('COMPLETE_HOOK') for commit, filename in results: working_dir = extract(filename, save_dir) try: build = build_sphinx(working_dir, env) except Exception: permalink = commit_map[commit] if not config.get('RECREATE_VIRTUALENV'): env = make_virtualenv(config, recreate=True) try: build = build_sphinx(working_dir, env) except Exception: if callable(complete_hook): complete_hook(commit, permalink, sys.exc_info()) continue if callable(complete_hook): complete_hook(commit, permalink, sys.exc_info()) continue result_dir = os.path.join(save_dir, commit) shutil.move(build, result_dir) logger.info('build complete: %s' % result_dir) if callable(complete_hook): complete_hook(commit, commit_map[commit], None) shutil.rmtree(working_dir) logger.info('working directory %s has removed' % working_dir) with open_head_file('w', config=config) as f: f.write(commit) logger.info('new head: %s', commits[0])
def download(files, settings): temp_dir = settings['temp_dir'] download_path = settings['download_path'] download = DownloadPool(settings) decode = Decode() pool = GreenPool(settings['connections']) progress_tracker = greenthread.spawn(show_progress) for file_ in files: # Check if file from subject exists. subject_filename = helper.get_filename_from(file_['file_subject']) if helper.file_exists(download_path, subject_filename): Tracker.total_size -= sum([i['segment_bytes'] for i in file_['segments']]) print subject_filename, 'already exists' continue # Download. for segment_path in pool.imap(download.download, file_['segments']): # Decode. if segment_path: tpool.execute(decode.decode, segment_path, temp_dir, download_path) if decode.tracker: print 'have broken files...' #return False print decode.tracker broken_files = decode.tracker.keys() for fname in broken_files: #print 'decoding', fname decode.join_files(fname, temp_dir, download_path) #progress_tracker.kill() progress_tracker.wait() # All OK. return 0
class WorkerManager(object): """ Methods of this class must be thread safe unless explicitly stated. """ __metaclass__ = ABCMeta daemon = True def __init__(self, games_url): """ :param thread_pool: """ self._data = _WorkerManagerData() self.games_url = games_url self._pool = GreenPool(size=3) super(WorkerManager, self).__init__() def get_persistent_state(self, player_id): """Get the persistent state for a worker.""" return None @abstractmethod def create_worker(self, player_id): """Create a worker.""" raise NotImplemented @abstractmethod def remove_worker(self, player_id): """Remove a worker for the given player.""" raise NotImplemented # TODO handle failure def spawn(self, game_id, game_data): # Kill worker LOGGER.info("Removing worker for game %s" % game_data['name']) self.remove_worker(game_id) # Spawn worker LOGGER.info("Spawning worker for game %s" % game_data['name']) game_data['GAME_API_URL'] = '{}{}/'.format(self.games_url, game_id) self.create_worker(game_id, game_data) def _parallel_map(self, func, *iterable_args): list(self._pool.imap(func, *iterable_args)) def run(self): while True: self.update() LOGGER.info("Sleeping") time.sleep(10) def update(self): try: LOGGER.info("Waking up") games = requests.get(self.games_url).json() except (requests.RequestException, ValueError) as err: LOGGER.error("Failed to obtain game data : %s", err) else: games_to_add = { id: games[id] for id in self._data.add_new_games(games.keys()) } LOGGER.debug("Need to add games: %s" % games_to_add) # Add missing games self._parallel_map(self.spawn, games_to_add.keys(), games_to_add.values()) # Delete extra games known_games = set(games.keys()) removed_user_ids = self._data.remove_unknown_games(known_games) LOGGER.debug("Removing users: %s" % removed_user_ids) self._parallel_map(self.remove_worker, removed_user_ids)
import httplib2 directly, httplib2 doesn't natively support cooperative yielding. monkey_patch httplib2 will make httplib2 green. ''' from eventlet.greenpool import GreenPool import eventlet import random import httplib2 # uncomment this line to green httplib2 # httplib2 = eventlet.import_patched('httplib2') def worker(url): print "worker " + str(random.random()) h = httplib2.Http() resp, content = h.request(url, "GET") return resp pool = GreenPool(size=10) results = pool.imap(worker, open("urls.txt", 'r')) for result in results: print result print "done...." pool.waitall()
class GameManager(object): """Methods of this class must be thread safe unless explicitly stated""" __metaclass__ = ABCMeta daemon = True def __init__(self, games_url): self._data = _GameManagerData() self.games_url = games_url self._pool = GreenPool(size=3) super(GameManager, self).__init__() @abstractmethod def create_game(self, game_id, game_data): """Creates a new game""" raise NotImplemented @abstractmethod def delete_game(self, game_id): """Deletes the given game""" raise NotImplemented def recreate_game(self, game_id, game_data): """Deletes and recreates the given game""" LOGGER.info("Deleting game {}".format(game_data["name"])) try: self.delete_game(game_id) except Exception as ex: LOGGER.error("Failed to delete game {}".format(game_data["name"])) LOGGER.exception(ex) LOGGER.info("Recreating game {}".format(game_data["name"])) try: game_data["GAME_API_URL"] = "{}{}/".format(self.games_url, game_id) self.create_game(game_id, game_data) except Exception as ex: LOGGER.error("Failed to create game {}".format(game_data["name"])) LOGGER.exception(ex) def update(self): try: LOGGER.info("Waking up") games = requests.get(self.games_url).json() except (requests.RequestException, ValueError) as ex: LOGGER.error("Failed to obtain game data") LOGGER.exception(ex) else: games_to_add = { id: games[id] for id in self._data.add_new_games(games.keys()) } LOGGER.debug("Need to add games: {}".format(games_to_add)) # Add missing games self._parallel_map(self.recreate_game, games_to_add.keys(), games_to_add.values()) # Delete extra games known_games = set(games.keys()) removed_game_ids = self._data.remove_unknown_games(known_games) LOGGER.debug("Removing games: {}".format(removed_game_ids)) self._parallel_map(self.delete_game, removed_game_ids) def get_persistent_state(self, player_id): """Get the persistent state of a game""" return None def run(self): while True: self.update() LOGGER.info("Sleeping") time.sleep(10) def _parallel_map(self, func, *iterable_args): list(self._pool.imap(func, *iterable_args))
class WorkerManager(threading.Thread): """ Methods of this class must be thread safe unless explicitly stated. """ daemon = True def __init__(self, game_state, users_url): """ :param thread_pool: """ self._data = _WorkerManagerData(game_state, {}) self.users_url = users_url self._pool = GreenPool(size=3) super(WorkerManager, self).__init__() def get_code(self, player_id): return self._data.get_code(player_id) def get_persistent_state(self, player_id): """Get the persistent state for a worker.""" return None def create_worker(self, player_id): """Create a worker.""" raise NotImplemented def remove_worker(self, player_id): """Remove a worker for the given player.""" raise NotImplemented # TODO handle failure def spawn(self, user): # Get persistent state from worker persistent_state = self.get_persistent_state(user['id']) # Kill worker LOGGER.info("Removing worker for user %s" % user['id']) self.remove_worker(user['id']) self._data.set_code(user) # Spawn worker LOGGER.info("Spawning worker for user %s" % user['id']) worker_url = self.create_worker(user['id']) # Add avatar back into game self._data.add_avatar(user, worker_url) LOGGER.info('Added user %s', user['id']) def _parallel_map(self, func, iterable_args): list(self._pool.imap(func, iterable_args)) def update(self): try: LOGGER.info("Waking up") game_data = requests.get(self.users_url).json() except (requests.RequestException, ValueError) as err: LOGGER.error("Failed to obtain game data : %s", err) else: game = game_data['main'] # Remove users with different code users_to_add = [] for user in game['users']: if self._data.remove_user_if_code_is_different(user): users_to_add.append(user) LOGGER.debug("Need to add users: %s" % [x['id'] for x in users_to_add]) # Add missing users self._parallel_map(self.spawn, users_to_add) # Delete extra users known_avatars = set(user['id'] for user in game['users']) removed_user_ids = self._data.remove_unknown_avatars(known_avatars) LOGGER.debug("Removing users: %s" % removed_user_ids) self._parallel_map(self.remove_worker, removed_user_ids) def run(self): while True: self.update() LOGGER.info("Sleeping") time.sleep(10)
def parallel_select(collection, func): gp = GreenPool() return list(gp.imap(func, collection))
import httplib2 directly, httplib2 doesn't natively support cooperative yielding. monkey_patch httplib2 will make httplib2 green. ''' from eventlet.greenpool import GreenPool import eventlet import random import httplib2 # uncomment this line to green httplib2 # httplib2 = eventlet.import_patched('httplib2') def worker(url): print "worker "+str(random.random()) h = httplib2.Http() resp, content = h.request(url, "GET") return resp pool = GreenPool(size=10) results = pool.imap(worker, open("urls.txt", 'r')) for result in results: print result print "done...." pool.waitall()
def _full_updatemacmap(configmanager): global vintage global _macmap global _nodesbymac global _switchportmap global _macsbyswitch with mapupdating: vintage = util.monotonic_time() # Clear all existing entries _macmap = {} _nodesbymac = {} _switchportmap = {} _macsbyswitch = {} if configmanager.tenant is not None: raise exc.ForbiddenRequest( 'Network topology not available to tenants') nodelocations = configmanager.get_node_attributes( configmanager.list_nodes(), ('net*.switch', 'net*.switchport')) switches = set([]) for node in nodelocations: cfg = nodelocations[node] for attr in cfg: if not attr.endswith('.switch') or 'value' not in cfg[attr]: continue curswitch = cfg[attr].get('value', None) if not curswitch: continue switches.add(curswitch) switchportattr = attr + 'port' if switchportattr in cfg: portname = cfg[switchportattr].get('value', '') if not portname: continue if curswitch not in _switchportmap: _switchportmap[curswitch] = {} if portname in _switchportmap[curswitch]: log.log({ 'error': 'Duplicate switch topology config ' 'for {0} and {1}'.format( node, _switchportmap[curswitch][portname]) }) _switchportmap[curswitch][portname] = None else: _switchportmap[curswitch][portname] = node switchcfg = configmanager.get_node_attributes( switches, ('secret.hardwaremanagementuser', 'secret.snmpcommunity', 'secret.hardwaremanagementpassword'), decrypt=True) switchauth = [] for switch in switches: if not switch: continue switchparms = switchcfg.get(switch, {}) user = None password = switchparms.get('secret.snmpcommunity', {}).get('value', None) if not password: password = switchparms.get('secret.hardwaremanagementpassword', {}).get('value', 'public') user = switchparms.get('secret.hardwaremanagementuser', {}).get('value', None) switchauth.append((switch, password, user)) pool = GreenPool() for ans in pool.imap(_map_switch, switchauth): vintage = util.monotonic_time() yield ans
class WorkerManager(threading.Thread): """ Methods of this class must be thread safe unless explicitly stated. """ daemon = True def __init__(self, game_state, users_url, port=5000): """ :param thread_pool: """ self._data = _WorkerManagerData(game_state, {}) self.users_url = users_url self._pool = GreenPool(size=3) self.port = port super(WorkerManager, self).__init__() def get_code(self, player_id): return self._data.get_code(player_id) def get_persistent_state(self, player_id): """Get the persistent state for a worker.""" return None def create_worker(self, player_id): """Create a worker.""" raise NotImplemented def remove_worker(self, player_id): """Remove a worker for the given player.""" raise NotImplemented # TODO handle failure def spawn(self, user): # Get persistent state from worker persistent_state = self.get_persistent_state(user['id']) # noqa: F841 # Kill worker LOGGER.info("Removing worker for user %s" % user['id']) self.remove_worker(user['id']) self._data.set_code(user) # Spawn worker LOGGER.info("Spawning worker for user %s" % user['id']) worker_url = self.create_worker(user['id']) # Add avatar back into game self._data.add_avatar(user, worker_url) LOGGER.info('Added user %s', user['id']) def _parallel_map(self, func, iterable_args): list(self._pool.imap(func, iterable_args)) def update(self): try: LOGGER.info("Waking up") game_data = requests.get(self.users_url).json() except (requests.RequestException, ValueError) as err: LOGGER.error("Failed to obtain game data : %s", err) else: game = game_data['main'] # Remove users with different code users_to_add = [] for user in game['users']: if self._data.remove_user_if_code_is_different(user): users_to_add.append(user) LOGGER.debug("Need to add users: %s" % [x['id'] for x in users_to_add]) # Add missing users self._parallel_map(self.spawn, users_to_add) # Delete extra users known_avatars = set(user['id'] for user in game['users']) removed_user_ids = self._data.remove_unknown_avatars(known_avatars) LOGGER.debug("Removing users: %s" % removed_user_ids) self._parallel_map(self.remove_worker, removed_user_ids) # Update main avatar self._data.set_main_avatar(game_data['main']['main_avatar']) def run(self): while True: self.update() LOGGER.info("Sleeping") time.sleep(10)
class WorkerManager(object): """ Methods of this class must be thread safe unless explicitly stated. """ __metaclass__ = ABCMeta daemon = True def __init__(self, games_url): """ :param thread_pool: """ self._data = _WorkerManagerData() self.games_url = games_url self._pool = GreenPool(size=3) super(WorkerManager, self).__init__() def get_persistent_state(self, player_id): """Get the persistent state for a worker.""" return None @abstractmethod def create_worker(self, player_id): """Create a worker.""" raise NotImplemented @abstractmethod def remove_worker(self, player_id): """Remove a worker for the given player.""" raise NotImplemented # TODO handle failure def spawn(self, game_id, game_data): # Kill worker LOGGER.info("Removing worker for game %s" % game_data['name']) self.remove_worker(game_id) # Spawn worker LOGGER.info("Spawning worker for game %s" % game_data['name']) game_data['GAME_API_URL'] = '{}{}/'.format(self.games_url, game_id) self.create_worker(game_id, game_data) def _parallel_map(self, func, *iterable_args): list(self._pool.imap(func, *iterable_args)) def run(self): while True: self.update() LOGGER.info("Sleeping") time.sleep(10) def update(self): try: LOGGER.info("Waking up") games = requests.get(self.games_url).json() except (requests.RequestException, ValueError) as err: LOGGER.error("Failed to obtain game data : %s", err) else: games_to_add = {id: games[id] for id in self._data.add_new_games(games.keys())} LOGGER.debug("Need to add games: %s" % games_to_add) # Add missing games self._parallel_map(self.spawn, games_to_add.keys(), games_to_add.values()) # Delete extra games known_games = set(games.keys()) removed_user_ids = self._data.remove_unknown_games(known_games) LOGGER.debug("Removing users: %s" % removed_user_ids) self._parallel_map(self.remove_worker, removed_user_ids)