def run(self):
last = None
off = 0
while True:
now = datetime.datetime.now()
sleep_time = 1.
if last is None:
diff = None
else:
diff = (now - last).total_seconds()
if in_window(self.window_start, self.window_end, now):
if diff is None or diff >= self.interval:
off = (diff or self.interval) - self.interval
logging.debug('On Interval (after %f seconds, off by %f).',
diff or 0, off)
self.callback(None)
last = now
else:
sleep_time = max(0.0001, min(1.,
self.interval - diff - off))
time.sleep(sleep_time)
def __exit__(self, type, value, traceback):
logging.debug("Exit pool.")
if self.join:
for thread in self.threads.values():
thread.join(self.timeout)
logging.debug("Exit joined %s. (%s)", thread.name,
"timed out" if thread.is_alive() else "ok")
total_time = time.time() - self.start_time
logging.info("Ran %i tasks in %f seconds (avg %f seconds per task).",
self.counter, total_time, self.avg_time)
def __exit__(self, type, value, traceback):
logging.debug("Exit pool.")
if self.join:
for thread in self.threads.values():
thread.join(self.timeout)
logging.debug("Exit joined %s. (%s)", thread.name,
"timed out" if thread.is_alive() else "ok")
total_time = time.time() - self.start_time
logging.info("Ran %i tasks in %f seconds (avg %f seconds per task).",
self.counter, total_time, self.avg_time)
def __call__(self, *args, **kwargs):
for retry in range(max_retries):
try:
logging.debug(u'RetryOnConflict, attempt %i of %i...',
retry + 1, max_retries)
kwargs['conflict'] = retry > 0
return (self.func)(*args, **kwargs)
# If we have a conflict, retry the entire operation
except HTTPClientError as e:
logging.warn(u'Update failed %s', str(e))
if e.response.status_code == 409: # etag violation
logging.warn(u'Retrying...')
else:
break
except Retry as e:
logging.warn(u'Retrying...')
def run(self):
last = None
off = 0
while True:
now = datetime.datetime.now()
sleep_time = 1.
if last is None:
diff = None
else:
diff = (now - last).total_seconds()
if in_window(self.window_start, self.window_end, now):
if diff is None or diff >= self.interval:
off = (diff or self.interval) - self.interval
logging.debug('On Interval (after %f seconds, off by %f).', diff or 0, off)
self.callback(None)
last = now
else:
sleep_time = max(0.0001, min(1., self.interval - diff - off))
time.sleep(sleep_time)
def spawn(self, worker, *args, **kwargs):
logging.debug("Start spawn.")
self.resource.acquire()
logging.debug("Acquired resource.")
def with_local_log_id(worker):
def wrapped(*args, **kwargs):
try:
logger = kwargs.pop('logger')
log_id = kwargs.pop('log_id')
logger.set_log_id(log_id)
except KeyError:
pass
except AttributeError:
pass
return worker(*args, **kwargs)
return wrapped
log_id = str(kwargs.get('log_id', 'x'))
worker_thread = threading.Thread(
target=with_local_log_id(worker),
name='worker_%s' % log_id,
args=args,
kwargs=kwargs,
)
worker_thread.daemon = True
control_thread = threading.Thread(
target=self._control,
name='control_%s' % log_id,
args=(worker_thread, ),
)
control_thread.daemon = True
with self.threads_lock:
self.threads[worker_thread.name] = control_thread
logging.debug("Spawn starts control thread %s.", control_thread.name)
control_thread.start()
def spawn(self, worker, *args, **kwargs):
logging.debug("Start spawn.")
self.resource.acquire()
logging.debug("Acquired resource.")
def with_local_log_id(worker):
def wrapped(*args, **kwargs):
try:
logger = kwargs.pop('logger')
log_id = kwargs.pop('log_id')
logger.set_log_id(log_id)
except KeyError:
pass
except AttributeError:
pass
return worker(*args, **kwargs)
return wrapped
log_id = str(kwargs.get('log_id', 'x'))
worker_thread = threading.Thread(
target=with_local_log_id(worker),
name='worker_%s' % log_id,
args=args,
kwargs=kwargs,
)
worker_thread.daemon = True
control_thread = threading.Thread(
target=self._control,
name='control_%s' % log_id,
args=(worker_thread, ),
)
control_thread.daemon = True
with self.threads_lock:
self.threads[worker_thread.name] = control_thread
logging.debug("Spawn starts control thread %s.", control_thread.name)
control_thread.start()
def _control(self, thread):
try:
logging.debug("Start control thread for %s.", thread.name)
start_time = time.time()
thread.start()
thread.join(self.timeout)
end_time = time.time()
logging.debug("Control thread joined %s. (%s)", thread.name,
"timed out" if thread.is_alive() else "ok")
with self.timing_lock:
self.avg_time = (self.avg_time + (end_time - start_time)) / 2
finally:
try:
with self.threads_lock:
del self.threads[thread.name]
finally:
self.resource.release()
logging.debug("Exit control thread for %s.", thread.name)
def _control(self, thread):
try:
logging.debug("Start control thread for %s.", thread.name)
start_time = time.time()
thread.start()
thread.join(self.timeout)
end_time = time.time()
logging.debug("Control thread joined %s. (%s)", thread.name,
"timed out" if thread.is_alive() else "ok")
with self.timing_lock:
self.avg_time = (self.avg_time + (end_time - start_time)) / 2
finally:
try:
with self.threads_lock:
del self.threads[thread.name]
finally:
self.resource.release()
logging.debug("Exit control thread for %s.", thread.name)
