def copy_files(self, container_id, path):
'''Returns a tarball of path from container_id'''
tarball = yield self.docker_client.copy(container_id, path)
raise gen.Return(tarball)
def async_value(value):
yield gen.moment # this ensures we actually return to the loop
raise gen.Return(value)
def _async_download(lcs, clobber=False):
client = AsyncHTTPClient(max_clients=10)
to_fetch = [l for l in lcs if clobber or not l.cache_exists]
responses = yield [client.fetch(HTTPRequest(l.url)) for l in to_fetch]
[handle_response(to_fetch[i], r) for i, r in enumerate(responses)]
raise gen.Return()
def version(client):
info = yield client.server_info()
raise gen.Return(_parse_version_string(info["version"]))
def get_command_line(client):
command_line = yield client.admin.command('getCmdLineOpts')
assert command_line['ok'] == 1, "getCmdLineOpts() failed"
raise gen.Return(command_line)
def getMeta(self, targetPath):
targetPath = url_escape(targetPath)
urlTemplate = "https://pcs.baidu.com/rest/2.0/pcs/file?method=meta&access_token=%s&path=%s" % (
self.tokens["access_token"], targetPath)
resp = yield self.fetch(urlTemplate)
raise gen.Return(json.loads(resp.body))
def receiveHandler(self, body):
rb_handler = ReceiveHandler(body)
yield rb_handler.handler()
result = rb_handler.get_result
raise gen.Return(json.dumps(result))
def geninc(x, delay=0.02):
yield gen.sleep(delay)
raise gen.Return(x + 1)
def tt():
yield gen.sleep(5)
raise gen.Return('hello')
def from_client(cls, client):
info = yield client.server_info()
if 'versionArray' in info:
raise gen.Return(cls.from_version_array(info['versionArray']))
raise gen.Return(cls.from_string(info['version']))
def co_add():
sessions = []
for kwargs in kwarg_list:
session = yield self.sm.create_session(**kwargs)
sessions.append(session)
raise gen.Return(sessions)
def range_query(self,
table_name,
column_names,
start_key,
end_key,
limit,
offset=0,
start_inclusive=True,
end_inclusive=True,
keys_only=False):
"""
Gets a dense range ordered by keys. Returns an ordered list of
a dictionary of [key:{column1:value1, column2:value2},...]
or a list of keys if keys only.
Args:
table_name: Name of table to access
column_names: Columns which get returned within the key range
start_key: String for which the query starts at
end_key: String for which the query ends at
limit: Maximum number of results to return
offset: Cuts off these many from the results [offset:]
start_inclusive: Boolean if results should include the start_key
end_inclusive: Boolean if results should include the end_key
keys_only: Boolean if to only keys and not values
Raises:
TypeError: If an argument passed in was not of the expected type.
AppScaleDBConnectionError: If the range_query could not be performed due
to an error with Cassandra.
Returns:
An ordered list of dictionaries of key=>columns/values
"""
if not isinstance(table_name, str):
raise TypeError('table_name must be a string')
if not isinstance(column_names, list):
raise TypeError('column_names must be a list')
if not isinstance(start_key, str):
raise TypeError('start_key must be a string')
if not isinstance(end_key, str):
raise TypeError('end_key must be a string')
if not isinstance(limit, (int, long)) and limit is not None:
raise TypeError('limit must be int, long, or NoneType')
if not isinstance(offset, (int, long)):
raise TypeError('offset must be int or long')
if start_inclusive:
gt_compare = '>='
else:
gt_compare = '>'
if end_inclusive:
lt_compare = '<='
else:
lt_compare = '<'
query_limit = ''
if limit is not None:
query_limit = 'LIMIT {}'.format(len(column_names) * limit)
statement = ('SELECT * FROM "{table}" WHERE '
'token({key}) {gt_compare} %s AND '
'token({key}) {lt_compare} %s AND '
'{column} IN %s '
'{limit} '
'ALLOW FILTERING').format(table=table_name,
key=ThriftColumn.KEY,
gt_compare=gt_compare,
lt_compare=lt_compare,
column=ThriftColumn.COLUMN_NAME,
limit=query_limit)
query = SimpleStatement(statement, retry_policy=BASIC_RETRIES)
parameters = (bytearray(start_key), bytearray(end_key),
ValueSequence(column_names))
try:
results = yield self.tornado_cassandra.execute(
query, parameters=parameters)
results_list = []
current_item = {}
current_key = None
for (key, column, value) in results:
if keys_only:
results_list.append(key)
continue
if key != current_key:
if current_item:
results_list.append({current_key: current_item})
current_item = {}
current_key = key
current_item[column] = value
if current_item:
results_list.append({current_key: current_item})
raise gen.Return(results_list[offset:])
except dbconstants.TRANSIENT_CASSANDRA_ERRORS:
message = 'Exception during range_query'
logging.exception(message)
raise AppScaleDBConnectionError(message)
def _execute_batch(self, op_name, nt_class, singleton_method, *req_entries,
**req_kwargs):
"""Asynchronously sends messages to the queue
"""
api_params = dict(QueueUrl=self.queue_url, )
req_entries = list(req_entries)
result = dict(
Successful=[],
Failed=[],
ResponseMetadata=[],
)
while req_entries:
entries = []
for entry in req_entries[:self.max_messages]:
assert isinstance(entry, nt_class)
# botocore expects dictionaries
entries.append(entry._asdict())
api_params['Entries'] = entries
try:
response = yield self._operate(op_name, api_params,
**req_kwargs)
except SQSError as err:
for entry in req_entries[:self.max_messages]:
try:
response = yield singleton_method(entry)
except SQSError as err:
log_msg = 'Too many SQS errors, retry failed: %s'
self.logger.error(log_msg, err)
result['Failed'].append(entry)
else:
result['Successful'].append(entry)
result['ResponseMetadata'].append(
response['ResponseMetadata'])
else:
for success in response.get('Successful', []):
# Populate our return data with objects passed in
# We want this to blow up, so that inconsistencies
# in the response are bubbled up
matching_items = [
sre for sre in req_entries if sre.Id == success['Id']
]
if len(matching_items) > 1:
message = 'Duplicate message IDs in batch: %s'
raise SQSError(
message=message % success['Id'],
code='9998',
error_type='ClientError',
detail='',
)
elif not matching_items:
message = 'No matching message ID for: %s'
raise SQSError(
message=message % success['Id'],
code='9999',
error_type='ClientError',
detail='',
)
result['Successful'].append(matching_items[0])
result['ResponseMetadata'].append(response['ResponseMetadata'])
for err in response.get('Failed', []):
entry = [
entry for entry in req_entries if entry.Id == err['Id']
][0]
try:
# This will include retry logic,
# up to self.retry_attempts
response = yield singleton_method(entry)
except SQSError as err:
log_msg = 'Too many SQS errors, retry failed: %s'
self.logger.error(log_msg, err)
result['Failed'].append(entry)
else:
result['Successful'].append(entry)
result['ResponseMetadata'].append(
response['ResponseMetadata'])
req_entries = req_entries[self.max_messages:]
raise gen.Return(BatchResponse(**result))
def create_notebook_server(self, base_path, container_name,
container_config):
'''Creates a notebook_server running off of `base_path`.
Returns the (container_id, ip, port) tuple in a Future.'''
port = container_config.container_port
app_log.debug(container_config)
# Assumes that the container_config.command is of a format like:
#
# ipython notebook --no-browser --port {port} --ip=0.0.0.0
# --NotebookApp.base_path=/{base_path}
# --NotebookApp.tornado_settings=\"{ \"template_path\": [ \"/srv/ga\",
# \"/srv/ipython/IPython/html\",
# \"/srv/ipython/IPython/html/templates\" ] }\""
#
# Important piece here is the parametrized base_path to let the
# underlying process know where the proxy is routing it.
rendered_command = container_config.command.format(base_path=base_path,
port=port)
command = ["/bin/sh", "-c", rendered_command]
host_config = dict(mem_limit=container_config.mem_limit)
host_config = create_host_config(**host_config)
cpu_shares = None
if container_config.cpu_shares:
# Some versions of Docker and docker-py won't cast from string to int
cpu_shares = int(container_config.cpu_shares)
resp = yield self._with_retries(self.docker_client.create_container,
image=container_config.image,
command=command,
host_config=host_config,
cpu_shares=cpu_shares,
name=container_name)
docker_warnings = resp.get('Warnings')
if docker_warnings is not None:
app_log.warn(docker_warnings)
container_id = resp['Id']
app_log.info("Created container {}".format(container_id))
port_bindings = {
container_config.container_port: (container_config.container_ip, )
}
yield self._with_retries(self.docker_client.start,
container_id,
port_bindings=port_bindings)
container_network = yield self._with_retries(
self.docker_client.port, container_id,
container_config.container_port)
host_port = container_network[0]['HostPort']
host_ip = container_network[0]['HostIp']
raise gen.Return((container_id, host_ip, int(host_port)))
def button_prompt(self, text, buttons):
action = yield self.channel.button_prompt(text, buttons)
raise gen.Return(action)
def handle_links(self):
yield self.semaphore.acquire()
newlink = yield links.get()
# Make sure we haven't visited this one
if newlink in visited_links:
self.semaphore.release()
raise gen.Return()
visited_links.add(newlink)
# use async client to fetch this url
client = AsyncHTTPClient()
tries = 3 # Give it 3 chances before putting it in failure
while tries:
response = yield client.fetch(newlink)
if response.code==200:
break
tries -= 1
# release the semaphore
self.semaphore.release()
if response.code!=200:
link_failures.append(newlink)
print "[FAILURE] - %s"%newlink
raise gen.Return()
# TODO: replace this with a report api
print "[VISITED] - %s"%newlink
# parse url to get the base url
components = urlparse.urlparse(newlink)
baseurl = components[0]+"://"+components[1]
path = components[2]
# parse the html with bs
soup = bs4.BeautifulSoup(response.body)
# extract valid links and put into links
a_tags = soup.find_all("a")
for tag in a_tags:
if "href" not in tag.attrs:
continue
href = tag['href']
if href.startswith("#"):
continue
if href.startswith("/"): # relative
href = baseurl+href
else:
if not path.endswith("/"):
path = path[:path.rfind("/")+1]
href = baseurl+"/"+path+href
if not self.link_regex.match(href):
continue
if href in visited_links:
continue
links.put(href)
print "NEWLINK:", href
# extract imgs and put into imageurls
img_tags = soup.find_all("img")
for tag in img_tags:
if "src" not in tag.attrs:
continue
src = tag['src']
if src.startswith("/"): # relative
src = baseurl+src
if not self.img_regex.match(src):
continue
if src in downloaded_images:
continue
imageurls.put(src)
print "NEW IMAGE:", src
# now the task is done
links.task_done()
def __call__(
self,
request,
headers=None,
timeout=None,
retry_on=None,
retry_limit=None,
shard_key=None,
trace=None,
hostport=None,
routing_delegate=None,
):
"""Make a Thrift TChannel request.
Returns a ``Response`` containing the return value of the Thrift
call (if any). If the remote server responded with a Thrift exception,
that exception is raised.
:param string request:
Request obtained by calling a method on service objects generated
by :py:func:`tchannel.thrift.load`.
:param dict headers:
Dictionary of header key-value pairs.
:param float timeout:
How long to wait (in seconds) before raising a ``TimeoutError`` -
this defaults to ``tchannel.glossary.DEFAULT_TIMEOUT``.
:param string hostport:
A 'host:port' value to use when making a request directly to a
TChannel service, bypassing Hyperbahn. This value takes precedence
over the ``hostport`` specified to
:py:func:`tchannel.thrift.load`.
:param string retry_on:
What events to retry on - valid values can be found in
``tchannel.retry``.
:param string retry_limit:
How many times to retry before
:param routing_delegate:
Name of a service to which the request router should forward the
request instead of the service specified in the call req.
:rtype: Response
"""
if not headers:
headers = {}
serializer = request.get_serializer()
# serialize
try:
headers = serializer.serialize_header(headers=headers)
except (AttributeError, TypeError):
raise ValueError(
'headers must be a map[string]string (a shallow dict'
' where keys and values are strings)')
body = serializer.serialize_body(request.call_args)
# TODO There's only one yield. Drop in favor of future+callback.
response = yield self._tchannel.call(
scheme=self.NAME,
service=request.service,
arg1=request.endpoint,
arg2=headers,
arg3=body,
timeout=timeout,
retry_on=retry_on,
retry_limit=retry_limit,
hostport=hostport or request.hostport,
shard_key=shard_key,
trace=trace,
routing_delegate=routing_delegate,
)
response.headers = serializer.deserialize_header(
headers=response.headers)
body = serializer.deserialize_body(body=response.body)
response.body = request.read_body(body)
raise gen.Return(response)
def dispatch(self, action, *args, **kwargs):
if not hasattr(self, action):
raise NotFound()
response = yield as_future(lambda: getattr(self, action)(*args, **kwargs))
raise gen.Return(response)
def f():
yield gen.Task(self.io_loop.add_callback)
raise gen.Return(42)
import logging
from tornado import gen
from common import *
from exception import VolumeTypeOperationFailed
__all__ = ["list_volume_type"]
LOG = logging.getLogger("system")
@gen.coroutine
def list_volume_type():
""" list volume types
"""
try:
volume_types = yield volume_type_list()
except Exception, e:
LOG.error("volume type list error: %s" % e)
raise VolumeTypeOperationFailed()
raise gen.Return(volume_types)
@gen.coroutine
def main():
pass
if __name__ == "__main__":
from tornado import ioloop
ioloop.IOLoop.current().run_sync(main)
def receiver(args):
logging.info("Enter creating_user")
logging.debug("args: %s", str(args))
raise gen.Return(True)
def resolve(self, host, port, *args, **kwargs):
if host == 'unix+restuser':
raise gen.Return([(socket.AF_UNIX, self.socket_path)])
result = yield self.resolver.resolve(host, port, *args, **kwargs)
raise gen.Return(result)
def at_least(client, min_version):
client_version = yield version(client)
raise gen.Return(client_version >= tuple(padded(min_version, 4)))
def client():
client = yield websocket_connect('ws://localhost:%s/ws' % port)
message = yield client.read_message()
client.write_message(json.dumps(string))
message = yield client.read_message()
raise gen.Return(json.loads(message))
def server_is_mongos(client):
ismaster_response = yield client.admin.command('ismaster')
raise gen.Return(ismaster_response.get('msg') == 'isdbgrid')
def get_keys(self, prefix=""):
keys = yield self.store.get_plan_keys(prefix)
raise gen.Return(keys)
def _add_or_update_endpoint(self, action, name, version, request_data):
'''
Add or update an endpoint
'''
self.logger.log(logging.DEBUG, f'Adding/updating model {name}...')
_name_checker = _compile(r'^[a-zA-Z0-9-_\s]+$')
if not isinstance(name, str):
msg = 'Endpoint name must be a string'
self.logger.log(logging.CRITICAL, msg)
raise TypeError(msg)
if not _name_checker.match(name):
raise gen.Return('endpoint name can only contain: a-z, A-Z, 0-9,'
' underscore, hyphens and spaces.')
if self.settings.get('add_or_updating_endpoint'):
msg = ('Another endpoint update is already in progress'
', please wait a while and try again')
self.logger.log(logging.CRITICAL, msg)
raise RuntimeError(msg)
request_uuid = random_uuid()
self.settings['add_or_updating_endpoint'] = request_uuid
try:
description = (request_data['description']
if 'description' in request_data else None)
if 'docstring' in request_data:
docstring = str(
bytes(request_data['docstring'],
"utf-8").decode('unicode_escape'))
else:
docstring = None
endpoint_type = (request_data['type']
if 'type' in request_data else None)
methods = (request_data['methods']
if 'methods' in request_data else [])
dependencies = (request_data['dependencies']
if 'dependencies' in request_data else None)
target = (request_data['target']
if 'target' in request_data else None)
schema = (request_data['schema']
if 'schema' in request_data else None)
src_path = (request_data['src_path']
if 'src_path' in request_data else None)
target_path = get_query_object_path(
self.settings[SettingsParameters.StateFilePath], name, version)
self.logger.log(logging.DEBUG,
f'Checking source path {src_path}...')
_path_checker = _compile(r'^[\\\:a-zA-Z0-9-_~\s/\.]+$')
# copy from staging
if src_path:
if not isinstance(request_data['src_path'], str):
raise gen.Return("src_path must be a string.")
if not _path_checker.match(src_path):
raise gen.Return(
'Endpoint source path name can only contain: '
'a-z, A-Z, 0-9, underscore, hyphens and spaces.')
yield self._copy_po_future(src_path, target_path)
elif endpoint_type != 'alias':
raise gen.Return("src_path is required to add/update an "
"endpoint.")
# alias special logic:
if endpoint_type == 'alias':
if not target:
raise gen.Return('Target is required for alias endpoint.')
dependencies = [target]
# update local config
try:
if action == 'add':
self.tabpy_state.add_endpoint(name=name,
description=description,
docstring=docstring,
endpoint_type=endpoint_type,
methods=methods,
dependencies=dependencies,
target=target,
schema=schema)
else:
self.tabpy_state.update_endpoint(
name=name,
description=description,
docstring=docstring,
endpoint_type=endpoint_type,
methods=methods,
dependencies=dependencies,
target=target,
schema=schema,
version=version)
except Exception as e:
raise gen.Return(f'Error when changing TabPy state: {e}')
on_state_change(self.settings, self.tabpy_state,
self.python_service, self.logger)
finally:
self.settings['add_or_updating_endpoint'] = None
def get_current_time(self):
raise gen.Return(timer.current())
def _fit(
model,
params,
X_train,
y_train,
X_test,
y_test,
additional_calls,
fit_params=None,
scorer=None,
random_state=None,
):
original_model = model
fit_params = fit_params or {}
client = default_client()
rng = check_random_state(random_state)
info = {}
models = {}
scores = {}
for ident, param in enumerate(params):
model = client.submit(_create_model, original_model, ident, **param)
info[ident] = []
models[ident] = model
# assume everything in fit_params is small and make it concrete
fit_params = yield client.compute(fit_params)
# Convert testing data into a single element on the cluster
# This assumes that it fits into memory on a single worker
if isinstance(X_test, da.Array):
X_test = client.compute(X_test)
else:
X_test = yield client.scatter(X_test)
if isinstance(y_test, da.Array):
y_test = client.compute(y_test)
else:
y_test = yield client.scatter(y_test)
# Convert to batches of delayed objects of numpy arrays
X_train, y_train = dask.persist(X_train, y_train)
X_train = sorted(futures_of(X_train), key=lambda f: f.key)
y_train = sorted(futures_of(y_train), key=lambda f: f.key)
assert len(X_train) == len(y_train)
# Order by which we process training data futures
order = []
def get_futures(partial_fit_calls):
""" Policy to get training data futures
Currently we compute once, and then keep in memory.
Presumably in the future we'll want to let data drop and recompute.
This function handles that policy internally, and also controls random
access to training data.
"""
# Shuffle blocks going forward to get uniform-but-random access
while partial_fit_calls >= len(order):
L = list(range(len(X_train)))
rng.shuffle(L)
order.extend(L)
j = order[partial_fit_calls]
return X_train[j], y_train[j]
# Submit initial partial_fit and score computations on first batch of data
X_future, y_future = get_futures(0)
X_future_2, y_future_2 = get_futures(1)
_models = {}
_scores = {}
_specs = {}
d_partial_fit = dask.delayed(_partial_fit)
d_score = dask.delayed(_score)
for ident, model in models.items():
model = d_partial_fit(model, X_future, y_future, fit_params)
score = d_score(model, X_test, y_test, scorer)
spec = d_partial_fit(model, X_future_2, y_future_2, fit_params)
_models[ident] = model
_scores[ident] = score
_specs[ident] = spec
_models, _scores, _specs = dask.persist(
_models, _scores, _specs, priority={tuple(_specs.values()): -1})
_models = {k: list(v.dask.values())[0] for k, v in _models.items()}
_scores = {k: list(v.dask.values())[0] for k, v in _scores.items()}
_specs = {k: list(v.dask.values())[0] for k, v in _specs.items()}
models.update(_models)
scores.update(_scores)
speculative = _specs
new_scores = list(_scores.values())
history = []
# async for future, result in seq:
while True:
metas = yield client.gather(new_scores)
for meta in metas:
ident = meta["model_id"]
info[ident].append(meta)
history.append(meta)
instructions = additional_calls(info)
bad = set(models) - set(instructions)
# Delete the futures of bad models. This cancels speculative tasks
for ident in bad:
del models[ident]
del scores[ident]
del info[ident]
if not any(instructions.values()):
break
_models = {}
_scores = {}
_specs = {}
for ident, k in instructions.items():
start = info[ident][-1]["partial_fit_calls"] + 1
if k:
k -= 1
model = speculative.pop(ident)
for i in range(k):
X_future, y_future = get_futures(start + i)
model = d_partial_fit(model, X_future, y_future,
fit_params)
score = d_score(model, X_test, y_test, scorer)
X_future, y_future = get_futures(start + k)
spec = d_partial_fit(model, X_future, y_future, fit_params)
_models[ident] = model
_scores[ident] = score
_specs[ident] = spec
_models2, _scores2, _specs2 = dask.persist(
_models, _scores, _specs, priority={tuple(_specs.values()): -1})
_models2 = {
k: v if isinstance(v, Future) else list(v.dask.values())[0]
for k, v in _models2.items()
}
_scores2 = {k: list(v.dask.values())[0] for k, v in _scores2.items()}
_specs2 = {k: list(v.dask.values())[0] for k, v in _specs2.items()}
models.update(_models2)
scores.update(_scores2)
speculative = _specs2
new_scores = list(_scores2.values())
models = {
k: client.submit(operator.getitem, v, 0)
for k, v in models.items()
}
yield wait(models)
scores = yield client.gather(scores)
best = max(scores.items(), key=lambda x: x[1]["score"])
info = defaultdict(list)
for h in history:
info[h["model_id"]].append(h)
info = dict(info)
raise gen.Return(Results(info, models, history, best))
def resolve(self, host, port, family=socket.AF_UNSPEC):
result = yield IOLoop.current().run_in_executor(
None, _resolve_addr, host, port, family)
raise gen.Return(result)
