def _call_backends(self, func, perm_checker, *args, **kwargs): pile = GreenPile(self.pool) reqid = request_id() for backend in (backend for backend in self.backends.values() if backend.permissions_ok(perm_checker)): pile.spawn(self._backend_call, func(backend), reqid, *args, **kwargs) for result in pile: if result: for value in result: yield value
def _run_all(self, executor, data): # Performs an operation locally and then mimics it on remote clients. # This function only returns data for the local instance, but will # wait for all remote instances to finish (and ignores their success or # failure). if not self.remote: return self._attempt(self.local, executor, data) pile = GreenPile(self.pool) ret = self.pool.spawn(self._attempt, self.local, executor, data) for server in self.remote: pile.spawn(self._attempt, server, executor, data) try: return ret.wait() except TooManyRetries, e: self.log.error(e.message) raise
def test_concurrency(): container = Mock() container.config = config container.service_name = "fooservice" entrypoint = DummyProvider() service_instance = Mock() def inject(worker_ctx): orm_session = OrmSession(DeclBase) orm_session.container = container return orm_session.acquire_injection(worker_ctx) # get injections concurrently pile = GreenPile() for _ in xrange(CONCURRENT_REQUESTS): worker_ctx = WorkerContext(container, service_instance, entrypoint) pile.spawn(inject, worker_ctx) results = set(pile) # injections should all be unique assert len(results) == CONCURRENT_REQUESTS
def setup_security_groups_support(self, reset_state=True): """ We will track security groups -> dvportgroup mapping though the use of custom attributes. A custom attribute must be defined initially and then queried at start for its id, which is then used to map to a particular value. """ service_content = self.connection.vim.retrieve_service_content() custom_field_manager = service_content.customFieldsManager result = vim_util.get_object_properties(self.connection.vim, custom_field_manager, "field") for field in result[0].propSet[0].val.CustomFieldDef: if SECURITY_GROUPS_ATTRIBUTE == field.name: LOG.debug("Found custom attribute for security groups with key %s", field.key) self.security_groups_attribute_key = field.key break else: LOG.debug("No custom attribute for security groups found, will create one.") field = self.connection.invoke_api(self.connection.vim, "AddCustomFieldDef", custom_field_manager, name=SECURITY_GROUPS_ATTRIBUTE, moType="DistributedVirtualPortgroup") LOG.debug("Created custom attribute for security groups with key %s", field.key) self.security_groups_attribute_key = field.key if reset_state: wait_pile = False pile = GreenPile(self.pool) if self.pool else GreenPile() # Will drop all security group rules from matching dvportgroups and remove empty portgroups for uuid, dvs in six.iteritems(self.uuid_dvs_map): sg_tagged_pgs = dvs.get_pg_per_sg_attribute(self.security_groups_attribute_key) for sg_set, pg in six.iteritems(sg_tagged_pgs): wait_pile = True if len(pg["vm"]) == 0: pile.spawn(dvs._delete_port_group, pg["ref"], pg["name"], ignore_in_use=True) else: port_config = dvs.builder.port_setting() port_config.blocked = dvs.builder.blocked(False) port_config.filterPolicy = dvs.builder.filter_policy([], None) pile.spawn(dvs.update_dvportgroup, pg["ref"], pg["configVersion"], port_config, name=dvs.dvportgroup_name(sg_set)) if wait_pile: for result in pile: pass
def _put_stream(self, account, container, obj_name, src, sysmeta, chunks, headers=None): global_checksum = hashlib.md5() total_bytes_transferred = 0 content_chunks = [] def _connect_put(chunk): raw_url = chunk["url"] parsed = urlparse(raw_url) try: chunk_path = parsed.path.split('/')[-1] hdrs = {} hdrs["transfer-encoding"] = "chunked" hdrs[chunk_headers["content_id"]] = sysmeta['id'] hdrs[chunk_headers["content_version"]] = sysmeta['version'] hdrs[chunk_headers["content_path"]] = utils.quote(obj_name) hdrs[chunk_headers["content_size"]] = sysmeta['content_length'] hdrs[chunk_headers["content_chunkmethod"]] = \ sysmeta['chunk_method'] hdrs[chunk_headers["content_mimetype"]] = sysmeta['mime_type'] hdrs[chunk_headers["content_policy"]] = sysmeta['policy'] hdrs[chunk_headers["content_chunksnb"]] = len(chunks) hdrs[chunk_headers["container_id"]] = \ utils.name2cid(account, container) hdrs[chunk_headers["chunk_pos"]] = chunk["pos"] hdrs[chunk_headers["chunk_id"]] = chunk_path with ConnectionTimeout(CONNECTION_TIMEOUT): conn = http_connect( parsed.netloc, 'PUT', parsed.path, hdrs) conn.chunk = chunk return conn except (Exception, Timeout): pass def _send_data(conn): while True: data = conn.queue.get() if not conn.failed: try: with ChunkWriteTimeout(CHUNK_TIMEOUT): conn.send(data) except (Exception, ChunkWriteTimeout): conn.failed = True conn.queue.task_done() for pos in range(len(chunks)): current_chunks = chunks[pos] pile = GreenPile(len(current_chunks)) for current_chunk in current_chunks: pile.spawn(_connect_put, current_chunk) conns = [conn for conn in pile if conn] min_conns = 1 if len(conns) < min_conns: raise exc.OioException("RAWX connection failure") bytes_transferred = 0 total_size = current_chunks[0]["size"] chunk_checksum = hashlib.md5() try: with utils.ContextPool(len(current_chunks)) as pool: for conn in conns: conn.failed = False conn.queue = Queue(PUT_QUEUE_DEPTH) pool.spawn(_send_data, conn) while True: remaining_bytes = total_size - bytes_transferred if WRITE_CHUNK_SIZE < remaining_bytes: read_size = WRITE_CHUNK_SIZE else: read_size = remaining_bytes with ClientReadTimeout(CLIENT_TIMEOUT): try: data = src.read(read_size) except (ValueError, IOError) as e: raise SourceReadError(str(e)) if len(data) == 0: for conn in conns: conn.queue.put('0\r\n\r\n') break chunk_checksum.update(data) global_checksum.update(data) bytes_transferred += len(data) for conn in conns: if not conn.failed: conn.queue.put('%x\r\n%s\r\n' % (len(data), data)) else: conns.remove(conn) if len(conns) < min_conns: raise exc.OioException("RAWX write failure") for conn in conns: if conn.queue.unfinished_tasks: conn.queue.join() conns = [conn for conn in conns if not conn.failed] except SourceReadError: raise except ClientReadTimeout: raise except Timeout as e: raise exc.OioTimeout(str(e)) except Exception as e: raise exc.OioException( "Exception during chunk write %s" % str(e)) final_chunks = [] for conn in conns: resp = conn.getresponse(True) if resp.status in (200, 201): conn.chunk["size"] = bytes_transferred final_chunks.append(conn.chunk) conn.close() if len(final_chunks) < min_conns: raise exc.OioException("RAWX write failure") checksum = chunk_checksum.hexdigest() for chunk in final_chunks: chunk["hash"] = checksum content_chunks += final_chunks total_bytes_transferred += bytes_transferred content_checksum = global_checksum.hexdigest() return content_chunks, total_bytes_transferred, content_checksum
def use_eventlet(): pool = GreenPool(100) pile = GreenPile(pool) for i in range(10): pile.spawn(do_large_computation) print pile.next()
def _pile(pool=None): if pool: return GreenPile(pool) else: return GreenPile()