def retryable(*args, **kwargs): start = time.time() for i in itertools.count(): try: return function(*args, **kwargs) except exc as e: log.info(fn.message(str(e))) if time.time() > start + timeout: raise e time.sleep(2 ** (i/2-1))
async def retry_function(*args, **kwargs): x = StopIteration for i, t in enumerate(timeouts): try: return await asyncio.wait_for(function(*args, **kwargs), timeout=t) except (asyncio.TimeoutError, TimeoutError) as e: x = log.info( fn.message('Timed out while trying function', exception=e, attempt=i, allowed=len(timeouts))) except exceptions as e: x = log.info( fn.message('Exception encountered while retrying function', exception=e, attempt=i)) await asyncio.sleep(t) raise x
async def _worker(self, name, ip, script, *, image, flavor): log.debug(fn.message('Submitting worker script', contents=script)) await self.instances[0][2] return await create_server(self.conn, name=name, image=image, flavor=flavor, ip=ip, network=self.network, user_data=script)
async def _scheduler(self, ip, port, flavor, volume): script = scheduler_script(ip, port, volume, python=self.python, preload=self.preload) log.debug(fn.message('Submitting scheduler script', contents=script)) return await create_server(self.conn, name=self.name, network=self.network, image=self.image, flavor=flavor, ip=ip, user_data=script)
def close_server(conn, server, graceful=True): ''' Close a single instance - `conn`: the OpenStack connection - `server`: the server or server id ''' server = getattr(server, 'id', server) if graceful: s = conn.compute.find_server(server) if s is not None and s.status != 'SHUTOFF': conn.compute.stop_server(s) s = conn.get_server(server) if s is None: return False try: return conn.delete_server(s, delete_ips=True) except openstack.exceptions.SDKException as e: log.warning(fn.message(str(e))) return conn.delete_server(s, delete_ips=False)
async def create(cls, name, image, flavor, *, ip=None, pool=None, net=None, nova=None, userdata=None, key=DEFAULT_OS_KEY, groups=DEFAULT_OS_GROUPS): '''openstack server create ${OS_USERNAME}-api-U-1 \ --flavor m1.tiny \ --image IMAGE-NAME \ --key-name ${OS_USERNAME}-api-key \ --security-group global-ssh \ --nic net-id=${OS_USERNAME}-api-net ''' nics = [{'net-id': Network(net).id}] log.info(fn.message('Creating OpenStack instance', image=str(image), flavor=str(flavor))) os = await async_exe(pool, as_nova(nova).servers.create, name=name, image=Image(image).os, flavor=Flavor(flavor).os, key_name=key, nics=nics, security_groups=groups, userdata=userdata) out = Instance(os) with fn.ErrorContext(log, 'Failed to create IP'): if ip is None: ip = await async_exe(pool, FloatingIP.create, 'public') else: ip = await ip with fn.ErrorContext(log, 'Failed to associate IP with server'): await async_exe(pool, retry_openstack(out.add_ip), ip) return out