def copy_files(self, container_id, path): '''Returns a tarball of path from container_id''' tarball = yield self.docker_client.copy(container_id, path) raise gen.Return(tarball)
def async_value(value): yield gen.moment # this ensures we actually return to the loop raise gen.Return(value)
def _async_download(lcs, clobber=False): client = AsyncHTTPClient(max_clients=10) to_fetch = [l for l in lcs if clobber or not l.cache_exists] responses = yield [client.fetch(HTTPRequest(l.url)) for l in to_fetch] [handle_response(to_fetch[i], r) for i, r in enumerate(responses)] raise gen.Return()
def version(client): info = yield client.server_info() raise gen.Return(_parse_version_string(info["version"]))
def get_command_line(client): command_line = yield client.admin.command('getCmdLineOpts') assert command_line['ok'] == 1, "getCmdLineOpts() failed" raise gen.Return(command_line)
def getMeta(self, targetPath): targetPath = url_escape(targetPath) urlTemplate = "https://pcs.baidu.com/rest/2.0/pcs/file?method=meta&access_token=%s&path=%s" % ( self.tokens["access_token"], targetPath) resp = yield self.fetch(urlTemplate) raise gen.Return(json.loads(resp.body))
def receiveHandler(self, body): rb_handler = ReceiveHandler(body) yield rb_handler.handler() result = rb_handler.get_result raise gen.Return(json.dumps(result))
def geninc(x, delay=0.02): yield gen.sleep(delay) raise gen.Return(x + 1)
def tt(): yield gen.sleep(5) raise gen.Return('hello')
def from_client(cls, client): info = yield client.server_info() if 'versionArray' in info: raise gen.Return(cls.from_version_array(info['versionArray'])) raise gen.Return(cls.from_string(info['version']))
def co_add(): sessions = [] for kwargs in kwarg_list: session = yield self.sm.create_session(**kwargs) sessions.append(session) raise gen.Return(sessions)
def range_query(self, table_name, column_names, start_key, end_key, limit, offset=0, start_inclusive=True, end_inclusive=True, keys_only=False): """ Gets a dense range ordered by keys. Returns an ordered list of a dictionary of [key:{column1:value1, column2:value2},...] or a list of keys if keys only. Args: table_name: Name of table to access column_names: Columns which get returned within the key range start_key: String for which the query starts at end_key: String for which the query ends at limit: Maximum number of results to return offset: Cuts off these many from the results [offset:] start_inclusive: Boolean if results should include the start_key end_inclusive: Boolean if results should include the end_key keys_only: Boolean if to only keys and not values Raises: TypeError: If an argument passed in was not of the expected type. AppScaleDBConnectionError: If the range_query could not be performed due to an error with Cassandra. Returns: An ordered list of dictionaries of key=>columns/values """ if not isinstance(table_name, str): raise TypeError('table_name must be a string') if not isinstance(column_names, list): raise TypeError('column_names must be a list') if not isinstance(start_key, str): raise TypeError('start_key must be a string') if not isinstance(end_key, str): raise TypeError('end_key must be a string') if not isinstance(limit, (int, long)) and limit is not None: raise TypeError('limit must be int, long, or NoneType') if not isinstance(offset, (int, long)): raise TypeError('offset must be int or long') if start_inclusive: gt_compare = '>=' else: gt_compare = '>' if end_inclusive: lt_compare = '<=' else: lt_compare = '<' query_limit = '' if limit is not None: query_limit = 'LIMIT {}'.format(len(column_names) * limit) statement = ('SELECT * FROM "{table}" WHERE ' 'token({key}) {gt_compare} %s AND ' 'token({key}) {lt_compare} %s AND ' '{column} IN %s ' '{limit} ' 'ALLOW FILTERING').format(table=table_name, key=ThriftColumn.KEY, gt_compare=gt_compare, lt_compare=lt_compare, column=ThriftColumn.COLUMN_NAME, limit=query_limit) query = SimpleStatement(statement, retry_policy=BASIC_RETRIES) parameters = (bytearray(start_key), bytearray(end_key), ValueSequence(column_names)) try: results = yield self.tornado_cassandra.execute( query, parameters=parameters) results_list = [] current_item = {} current_key = None for (key, column, value) in results: if keys_only: results_list.append(key) continue if key != current_key: if current_item: results_list.append({current_key: current_item}) current_item = {} current_key = key current_item[column] = value if current_item: results_list.append({current_key: current_item}) raise gen.Return(results_list[offset:]) except dbconstants.TRANSIENT_CASSANDRA_ERRORS: message = 'Exception during range_query' logging.exception(message) raise AppScaleDBConnectionError(message)
def _execute_batch(self, op_name, nt_class, singleton_method, *req_entries, **req_kwargs): """Asynchronously sends messages to the queue """ api_params = dict(QueueUrl=self.queue_url, ) req_entries = list(req_entries) result = dict( Successful=[], Failed=[], ResponseMetadata=[], ) while req_entries: entries = [] for entry in req_entries[:self.max_messages]: assert isinstance(entry, nt_class) # botocore expects dictionaries entries.append(entry._asdict()) api_params['Entries'] = entries try: response = yield self._operate(op_name, api_params, **req_kwargs) except SQSError as err: for entry in req_entries[:self.max_messages]: try: response = yield singleton_method(entry) except SQSError as err: log_msg = 'Too many SQS errors, retry failed: %s' self.logger.error(log_msg, err) result['Failed'].append(entry) else: result['Successful'].append(entry) result['ResponseMetadata'].append( response['ResponseMetadata']) else: for success in response.get('Successful', []): # Populate our return data with objects passed in # We want this to blow up, so that inconsistencies # in the response are bubbled up matching_items = [ sre for sre in req_entries if sre.Id == success['Id'] ] if len(matching_items) > 1: message = 'Duplicate message IDs in batch: %s' raise SQSError( message=message % success['Id'], code='9998', error_type='ClientError', detail='', ) elif not matching_items: message = 'No matching message ID for: %s' raise SQSError( message=message % success['Id'], code='9999', error_type='ClientError', detail='', ) result['Successful'].append(matching_items[0]) result['ResponseMetadata'].append(response['ResponseMetadata']) for err in response.get('Failed', []): entry = [ entry for entry in req_entries if entry.Id == err['Id'] ][0] try: # This will include retry logic, # up to self.retry_attempts response = yield singleton_method(entry) except SQSError as err: log_msg = 'Too many SQS errors, retry failed: %s' self.logger.error(log_msg, err) result['Failed'].append(entry) else: result['Successful'].append(entry) result['ResponseMetadata'].append( response['ResponseMetadata']) req_entries = req_entries[self.max_messages:] raise gen.Return(BatchResponse(**result))
def create_notebook_server(self, base_path, container_name, container_config): '''Creates a notebook_server running off of `base_path`. Returns the (container_id, ip, port) tuple in a Future.''' port = container_config.container_port app_log.debug(container_config) # Assumes that the container_config.command is of a format like: # # ipython notebook --no-browser --port {port} --ip=0.0.0.0 # --NotebookApp.base_path=/{base_path} # --NotebookApp.tornado_settings=\"{ \"template_path\": [ \"/srv/ga\", # \"/srv/ipython/IPython/html\", # \"/srv/ipython/IPython/html/templates\" ] }\"" # # Important piece here is the parametrized base_path to let the # underlying process know where the proxy is routing it. rendered_command = container_config.command.format(base_path=base_path, port=port) command = ["/bin/sh", "-c", rendered_command] host_config = dict(mem_limit=container_config.mem_limit) host_config = create_host_config(**host_config) cpu_shares = None if container_config.cpu_shares: # Some versions of Docker and docker-py won't cast from string to int cpu_shares = int(container_config.cpu_shares) resp = yield self._with_retries(self.docker_client.create_container, image=container_config.image, command=command, host_config=host_config, cpu_shares=cpu_shares, name=container_name) docker_warnings = resp.get('Warnings') if docker_warnings is not None: app_log.warn(docker_warnings) container_id = resp['Id'] app_log.info("Created container {}".format(container_id)) port_bindings = { container_config.container_port: (container_config.container_ip, ) } yield self._with_retries(self.docker_client.start, container_id, port_bindings=port_bindings) container_network = yield self._with_retries( self.docker_client.port, container_id, container_config.container_port) host_port = container_network[0]['HostPort'] host_ip = container_network[0]['HostIp'] raise gen.Return((container_id, host_ip, int(host_port)))
def button_prompt(self, text, buttons): action = yield self.channel.button_prompt(text, buttons) raise gen.Return(action)
def handle_links(self): yield self.semaphore.acquire() newlink = yield links.get() # Make sure we haven't visited this one if newlink in visited_links: self.semaphore.release() raise gen.Return() visited_links.add(newlink) # use async client to fetch this url client = AsyncHTTPClient() tries = 3 # Give it 3 chances before putting it in failure while tries: response = yield client.fetch(newlink) if response.code==200: break tries -= 1 # release the semaphore self.semaphore.release() if response.code!=200: link_failures.append(newlink) print "[FAILURE] - %s"%newlink raise gen.Return() # TODO: replace this with a report api print "[VISITED] - %s"%newlink # parse url to get the base url components = urlparse.urlparse(newlink) baseurl = components[0]+"://"+components[1] path = components[2] # parse the html with bs soup = bs4.BeautifulSoup(response.body) # extract valid links and put into links a_tags = soup.find_all("a") for tag in a_tags: if "href" not in tag.attrs: continue href = tag['href'] if href.startswith("#"): continue if href.startswith("/"): # relative href = baseurl+href else: if not path.endswith("/"): path = path[:path.rfind("/")+1] href = baseurl+"/"+path+href if not self.link_regex.match(href): continue if href in visited_links: continue links.put(href) print "NEWLINK:", href # extract imgs and put into imageurls img_tags = soup.find_all("img") for tag in img_tags: if "src" not in tag.attrs: continue src = tag['src'] if src.startswith("/"): # relative src = baseurl+src if not self.img_regex.match(src): continue if src in downloaded_images: continue imageurls.put(src) print "NEW IMAGE:", src # now the task is done links.task_done()
def __call__( self, request, headers=None, timeout=None, retry_on=None, retry_limit=None, shard_key=None, trace=None, hostport=None, routing_delegate=None, ): """Make a Thrift TChannel request. Returns a ``Response`` containing the return value of the Thrift call (if any). If the remote server responded with a Thrift exception, that exception is raised. :param string request: Request obtained by calling a method on service objects generated by :py:func:`tchannel.thrift.load`. :param dict headers: Dictionary of header key-value pairs. :param float timeout: How long to wait (in seconds) before raising a ``TimeoutError`` - this defaults to ``tchannel.glossary.DEFAULT_TIMEOUT``. :param string hostport: A 'host:port' value to use when making a request directly to a TChannel service, bypassing Hyperbahn. This value takes precedence over the ``hostport`` specified to :py:func:`tchannel.thrift.load`. :param string retry_on: What events to retry on - valid values can be found in ``tchannel.retry``. :param string retry_limit: How many times to retry before :param routing_delegate: Name of a service to which the request router should forward the request instead of the service specified in the call req. :rtype: Response """ if not headers: headers = {} serializer = request.get_serializer() # serialize try: headers = serializer.serialize_header(headers=headers) except (AttributeError, TypeError): raise ValueError( 'headers must be a map[string]string (a shallow dict' ' where keys and values are strings)') body = serializer.serialize_body(request.call_args) # TODO There's only one yield. Drop in favor of future+callback. response = yield self._tchannel.call( scheme=self.NAME, service=request.service, arg1=request.endpoint, arg2=headers, arg3=body, timeout=timeout, retry_on=retry_on, retry_limit=retry_limit, hostport=hostport or request.hostport, shard_key=shard_key, trace=trace, routing_delegate=routing_delegate, ) response.headers = serializer.deserialize_header( headers=response.headers) body = serializer.deserialize_body(body=response.body) response.body = request.read_body(body) raise gen.Return(response)
def dispatch(self, action, *args, **kwargs): if not hasattr(self, action): raise NotFound() response = yield as_future(lambda: getattr(self, action)(*args, **kwargs)) raise gen.Return(response)
def f(): yield gen.Task(self.io_loop.add_callback) raise gen.Return(42)
import logging from tornado import gen from common import * from exception import VolumeTypeOperationFailed __all__ = ["list_volume_type"] LOG = logging.getLogger("system") @gen.coroutine def list_volume_type(): """ list volume types """ try: volume_types = yield volume_type_list() except Exception, e: LOG.error("volume type list error: %s" % e) raise VolumeTypeOperationFailed() raise gen.Return(volume_types) @gen.coroutine def main(): pass if __name__ == "__main__": from tornado import ioloop ioloop.IOLoop.current().run_sync(main)
def receiver(args): logging.info("Enter creating_user") logging.debug("args: %s", str(args)) raise gen.Return(True)
def resolve(self, host, port, *args, **kwargs): if host == 'unix+restuser': raise gen.Return([(socket.AF_UNIX, self.socket_path)]) result = yield self.resolver.resolve(host, port, *args, **kwargs) raise gen.Return(result)
def at_least(client, min_version): client_version = yield version(client) raise gen.Return(client_version >= tuple(padded(min_version, 4)))
def client(): client = yield websocket_connect('ws://localhost:%s/ws' % port) message = yield client.read_message() client.write_message(json.dumps(string)) message = yield client.read_message() raise gen.Return(json.loads(message))
def server_is_mongos(client): ismaster_response = yield client.admin.command('ismaster') raise gen.Return(ismaster_response.get('msg') == 'isdbgrid')
def get_keys(self, prefix=""): keys = yield self.store.get_plan_keys(prefix) raise gen.Return(keys)
def _add_or_update_endpoint(self, action, name, version, request_data): ''' Add or update an endpoint ''' self.logger.log(logging.DEBUG, f'Adding/updating model {name}...') _name_checker = _compile(r'^[a-zA-Z0-9-_\s]+$') if not isinstance(name, str): msg = 'Endpoint name must be a string' self.logger.log(logging.CRITICAL, msg) raise TypeError(msg) if not _name_checker.match(name): raise gen.Return('endpoint name can only contain: a-z, A-Z, 0-9,' ' underscore, hyphens and spaces.') if self.settings.get('add_or_updating_endpoint'): msg = ('Another endpoint update is already in progress' ', please wait a while and try again') self.logger.log(logging.CRITICAL, msg) raise RuntimeError(msg) request_uuid = random_uuid() self.settings['add_or_updating_endpoint'] = request_uuid try: description = (request_data['description'] if 'description' in request_data else None) if 'docstring' in request_data: docstring = str( bytes(request_data['docstring'], "utf-8").decode('unicode_escape')) else: docstring = None endpoint_type = (request_data['type'] if 'type' in request_data else None) methods = (request_data['methods'] if 'methods' in request_data else []) dependencies = (request_data['dependencies'] if 'dependencies' in request_data else None) target = (request_data['target'] if 'target' in request_data else None) schema = (request_data['schema'] if 'schema' in request_data else None) src_path = (request_data['src_path'] if 'src_path' in request_data else None) target_path = get_query_object_path( self.settings[SettingsParameters.StateFilePath], name, version) self.logger.log(logging.DEBUG, f'Checking source path {src_path}...') _path_checker = _compile(r'^[\\\:a-zA-Z0-9-_~\s/\.]+$') # copy from staging if src_path: if not isinstance(request_data['src_path'], str): raise gen.Return("src_path must be a string.") if not _path_checker.match(src_path): raise gen.Return( 'Endpoint source path name can only contain: ' 'a-z, A-Z, 0-9, underscore, hyphens and spaces.') yield self._copy_po_future(src_path, target_path) elif endpoint_type != 'alias': raise gen.Return("src_path is required to add/update an " "endpoint.") # alias special logic: if endpoint_type == 'alias': if not target: raise gen.Return('Target is required for alias endpoint.') dependencies = [target] # update local config try: if action == 'add': self.tabpy_state.add_endpoint(name=name, description=description, docstring=docstring, endpoint_type=endpoint_type, methods=methods, dependencies=dependencies, target=target, schema=schema) else: self.tabpy_state.update_endpoint( name=name, description=description, docstring=docstring, endpoint_type=endpoint_type, methods=methods, dependencies=dependencies, target=target, schema=schema, version=version) except Exception as e: raise gen.Return(f'Error when changing TabPy state: {e}') on_state_change(self.settings, self.tabpy_state, self.python_service, self.logger) finally: self.settings['add_or_updating_endpoint'] = None
def get_current_time(self): raise gen.Return(timer.current())
def _fit( model, params, X_train, y_train, X_test, y_test, additional_calls, fit_params=None, scorer=None, random_state=None, ): original_model = model fit_params = fit_params or {} client = default_client() rng = check_random_state(random_state) info = {} models = {} scores = {} for ident, param in enumerate(params): model = client.submit(_create_model, original_model, ident, **param) info[ident] = [] models[ident] = model # assume everything in fit_params is small and make it concrete fit_params = yield client.compute(fit_params) # Convert testing data into a single element on the cluster # This assumes that it fits into memory on a single worker if isinstance(X_test, da.Array): X_test = client.compute(X_test) else: X_test = yield client.scatter(X_test) if isinstance(y_test, da.Array): y_test = client.compute(y_test) else: y_test = yield client.scatter(y_test) # Convert to batches of delayed objects of numpy arrays X_train, y_train = dask.persist(X_train, y_train) X_train = sorted(futures_of(X_train), key=lambda f: f.key) y_train = sorted(futures_of(y_train), key=lambda f: f.key) assert len(X_train) == len(y_train) # Order by which we process training data futures order = [] def get_futures(partial_fit_calls): """ Policy to get training data futures Currently we compute once, and then keep in memory. Presumably in the future we'll want to let data drop and recompute. This function handles that policy internally, and also controls random access to training data. """ # Shuffle blocks going forward to get uniform-but-random access while partial_fit_calls >= len(order): L = list(range(len(X_train))) rng.shuffle(L) order.extend(L) j = order[partial_fit_calls] return X_train[j], y_train[j] # Submit initial partial_fit and score computations on first batch of data X_future, y_future = get_futures(0) X_future_2, y_future_2 = get_futures(1) _models = {} _scores = {} _specs = {} d_partial_fit = dask.delayed(_partial_fit) d_score = dask.delayed(_score) for ident, model in models.items(): model = d_partial_fit(model, X_future, y_future, fit_params) score = d_score(model, X_test, y_test, scorer) spec = d_partial_fit(model, X_future_2, y_future_2, fit_params) _models[ident] = model _scores[ident] = score _specs[ident] = spec _models, _scores, _specs = dask.persist( _models, _scores, _specs, priority={tuple(_specs.values()): -1}) _models = {k: list(v.dask.values())[0] for k, v in _models.items()} _scores = {k: list(v.dask.values())[0] for k, v in _scores.items()} _specs = {k: list(v.dask.values())[0] for k, v in _specs.items()} models.update(_models) scores.update(_scores) speculative = _specs new_scores = list(_scores.values()) history = [] # async for future, result in seq: while True: metas = yield client.gather(new_scores) for meta in metas: ident = meta["model_id"] info[ident].append(meta) history.append(meta) instructions = additional_calls(info) bad = set(models) - set(instructions) # Delete the futures of bad models. This cancels speculative tasks for ident in bad: del models[ident] del scores[ident] del info[ident] if not any(instructions.values()): break _models = {} _scores = {} _specs = {} for ident, k in instructions.items(): start = info[ident][-1]["partial_fit_calls"] + 1 if k: k -= 1 model = speculative.pop(ident) for i in range(k): X_future, y_future = get_futures(start + i) model = d_partial_fit(model, X_future, y_future, fit_params) score = d_score(model, X_test, y_test, scorer) X_future, y_future = get_futures(start + k) spec = d_partial_fit(model, X_future, y_future, fit_params) _models[ident] = model _scores[ident] = score _specs[ident] = spec _models2, _scores2, _specs2 = dask.persist( _models, _scores, _specs, priority={tuple(_specs.values()): -1}) _models2 = { k: v if isinstance(v, Future) else list(v.dask.values())[0] for k, v in _models2.items() } _scores2 = {k: list(v.dask.values())[0] for k, v in _scores2.items()} _specs2 = {k: list(v.dask.values())[0] for k, v in _specs2.items()} models.update(_models2) scores.update(_scores2) speculative = _specs2 new_scores = list(_scores2.values()) models = { k: client.submit(operator.getitem, v, 0) for k, v in models.items() } yield wait(models) scores = yield client.gather(scores) best = max(scores.items(), key=lambda x: x[1]["score"]) info = defaultdict(list) for h in history: info[h["model_id"]].append(h) info = dict(info) raise gen.Return(Results(info, models, history, best))
def resolve(self, host, port, family=socket.AF_UNSPEC): result = yield IOLoop.current().run_in_executor( None, _resolve_addr, host, port, family) raise gen.Return(result)