def __init__(self, thread_pool_size=1000):
self.tg = greenpool.GreenPool(size=thread_pool_size)
self.exc = None
self.exc_stacktrace = None
self.failed_thread = None
self.threads = 0
self.cv = threading.Condition()
def __init__(self):
super(_GreenFuture, self).__init__()
# NOTE(harlowja): replace the built-in condition with a greenthread
# compatible one so that when getting the result of this future the
# functions will correctly yield to eventlet. If this is not done then
# waiting on the future never actually causes the greenthreads to run
# and thus you wait for infinity.
if not patcher.is_monkey_patched('threading'):
self._condition = gthreading.Condition()
def __init__(self, *args, **kwargs):
self.vms_conn = kwargs.pop('vmsconn', None)
self._init_vms()
self.network_api = network.API()
self.gridcentric_api = API()
self.compute_manager = compute_manager.ComputeManager()
# Use an eventlet green thread condition lock instead of the regular threading module. This
# is required for eventlet threads because they essentially run on a single system thread.
# All of the green threads will share the same base lock, defeating the point of using the
# it. Since the main threading module is not monkey patched we cannot use it directly.
self.cond = gthreading.Condition()
self.locked_instances = {}
super(GridCentricManager, self).__init__(service_name="gridcentric",
*args,
**kwargs)
def condition_object(*args, **kwargs):
return greenthreading.Condition(*args, **kwargs)
def make_condition():
return threading.Condition()
class Lifetime(Renderer):
"""
Wraps a Renderer instance and limits the maximum time a single render
instance can live, and also shuts down a process if it has been unused for
a specified time period.
PhantomJS can be configured to use an in-memory browser cache which can
grow without bounds. The Lifetime decorator is useful for controlling this
growth by limiting the amount of time a single Renderer can live, and
releasing its accumulated resources.
For long or continual running of the rendering process, it is highly
recommended to wrap the Renderer with the Lifetime decorator to prevent
eventual OOM.
"""
_delegate = None
_last_render_time = None
_start_time = None
_running = False
_thread = None
_condition = threading.Condition(threading.RLock())
_lock = threading.RLock()
def __init__(self, renderer):
self._delegate = renderer
self.config = copy.deepcopy(LIFETIME)
self.config.update(renderer.get_config())
self._logger = logging.getLogger(u'LifetimeDecorator')
def get_config(self):
return self.config
def render(self,
url,
html=None,
img_format='PNG',
width=1280,
height=1024,
page_load_timeout=None,
user_agent=None,
headers=None,
cookies=None,
html_encoding=u'utf-8',
http_proxy=None):
with self._lock:
self._last_render_time = time.time()
if self._start_time is None:
self._start_time = self._last_render_time
if not self._running:
self._startup()
return self._delegate.render(url, html, img_format, width, height,
page_load_timeout, user_agent,
headers, cookies, html_encoding,
http_proxy)
def _startup(self):
self._running = True
self._thread = eventlet.spawn(self._lifetime_monitor)
def shutdown(self, timeout=None):
self._running = False
if self._condition is not None:
# Wake the monitor thread
with self._condition:
self._condition.notify()
if self._thread is not None:
try:
self._thread.kill()
except Exception:
pass
self._delegate.shutdown(timeout)
def _lifetime_monitor(self):
while self._running:
now = time.time()
sleep_delta = None
with self._lock:
if self._last_render_time is not None:
idle_target = self.config[
'idle_shutdown_sec'] + self._last_render_time
if now >= idle_target:
self._logger.info(u"Shutting down idle renderer.")
self._last_render_time = None
self._start_time = None
self._delegate.shutdown()
self._running = False
break
else:
sleep_delta = idle_target - now
if self._start_time is not None:
expired_target = self.config[
'max_lifetime_sec'] + self._start_time
if now >= expired_target:
self._logger.info(
u"Shutting down renderer which reached max lifetime."
)
self._last_render_time = None
self._start_time = None
self._delegate.shutdown()
self._running = False
break
else:
if sleep_delta is not None:
sleep_delta = min(sleep_delta,
expired_target - now)
else:
sleep_delta = expired_target - now
if sleep_delta is not None:
with self._condition:
if self._running:
self._condition.wait(sleep_delta)
