def retryable(*args, **kwargs):
start = time.time()
for i in itertools.count():
try:
return function(*args, **kwargs)
except exc as e:
log.info(fn.message(str(e)))
if time.time() > start + timeout: raise e
time.sleep(2 ** (i/2-1))
async def retry_function(*args, **kwargs):
x = StopIteration
for i, t in enumerate(timeouts):
try:
return await asyncio.wait_for(function(*args, **kwargs),
timeout=t)
except (asyncio.TimeoutError, TimeoutError) as e:
x = log.info(
fn.message('Timed out while trying function',
exception=e,
attempt=i,
allowed=len(timeouts)))
except exceptions as e:
x = log.info(
fn.message('Exception encountered while retrying function',
exception=e,
attempt=i))
await asyncio.sleep(t)
raise x
async def _worker(self, name, ip, script, *, image, flavor):
log.debug(fn.message('Submitting worker script', contents=script))
await self.instances[0][2]
return await create_server(self.conn,
name=name,
image=image,
flavor=flavor,
ip=ip,
network=self.network,
user_data=script)
async def _scheduler(self, ip, port, flavor, volume):
script = scheduler_script(ip,
port,
volume,
python=self.python,
preload=self.preload)
log.debug(fn.message('Submitting scheduler script', contents=script))
return await create_server(self.conn,
name=self.name,
network=self.network,
image=self.image,
flavor=flavor,
ip=ip,
user_data=script)
def close_server(conn, server, graceful=True):
'''
Close a single instance
- `conn`: the OpenStack connection
- `server`: the server or server id
'''
server = getattr(server, 'id', server)
if graceful:
s = conn.compute.find_server(server)
if s is not None and s.status != 'SHUTOFF':
conn.compute.stop_server(s)
s = conn.get_server(server)
if s is None:
return False
try:
return conn.delete_server(s, delete_ips=True)
except openstack.exceptions.SDKException as e:
log.warning(fn.message(str(e)))
return conn.delete_server(s, delete_ips=False)
async def create(cls, name, image, flavor, *, ip=None, pool=None, net=None, nova=None,
userdata=None, key=DEFAULT_OS_KEY, groups=DEFAULT_OS_GROUPS):
'''openstack server create ${OS_USERNAME}-api-U-1 \
--flavor m1.tiny \
--image IMAGE-NAME \
--key-name ${OS_USERNAME}-api-key \
--security-group global-ssh \
--nic net-id=${OS_USERNAME}-api-net
'''
nics = [{'net-id': Network(net).id}]
log.info(fn.message('Creating OpenStack instance', image=str(image), flavor=str(flavor)))
os = await async_exe(pool, as_nova(nova).servers.create, name=name, image=Image(image).os,
flavor=Flavor(flavor).os, key_name=key, nics=nics, security_groups=groups, userdata=userdata)
out = Instance(os)
with fn.ErrorContext(log, 'Failed to create IP'):
if ip is None:
ip = await async_exe(pool, FloatingIP.create, 'public')
else:
ip = await ip
with fn.ErrorContext(log, 'Failed to associate IP with server'):
await async_exe(pool, retry_openstack(out.add_ip), ip)
return out