def isJobRegistered(self, name):
if self.function_cache_time:
for connection in self.gearman.active_connections:
if connection.connect_time > self.function_cache_time:
self.function_cache = set()
self.function_cache_time = 0
break
if name in self.function_cache:
self.log.debug("Function %s is registered" % name)
return True
if ((time.time() - self.function_cache_time) <
self.negative_function_cache_ttl):
self.log.debug("Function %s is not registered "
"(negative ttl in effect)" % name)
return False
self.function_cache_time = time.time()
for connection in self.gearman.active_connections:
try:
req = gear.StatusAdminRequest()
connection.sendAdminRequest(req)
except Exception:
self.log.exception("Exception while checking functions")
continue
for line in req.response.split('\n'):
parts = [x.strip() for x in line.split()]
if not parts or parts[0] == '.':
continue
self.function_cache.add(parts[0])
if name in self.function_cache:
self.log.debug("Function %s is registered" % name)
return True
self.log.debug("Function %s is not registered" % name)
return False
def test_readPacket_admin(self):
response = b'test\t0\t0\t1\n.\n'
req = gear.StatusAdminRequest()
self.socket._set_data([response])
self.conn.admin_requests.append(req)
r1 = self.conn.readPacket()
self.assertEqual(r1.response, response)
self.assertEndOfData()
def test_partial_packet(self):
req = gear.StatusAdminRequest()
for i in range(len(self.response) - len(self.remainder)):
ret = req.isComplete(self.response[:i])
self.assertFalse(ret[0])
self.assertIsNone(ret[1])
ret = req.isComplete(self.response)
self.assertTrue(ret[0])
self.assertEqual(ret[1], self.remainder)
def getDemand(self):
needed_workers = {}
job_worker_map = {}
unspecified_jobs = {}
for connection in self.active_connections:
try:
req = gear.StatusAdminRequest()
connection.sendAdminRequest(req)
except Exception:
self._lostConnection(connection)
continue
# demand comes in the format build:function:node_type total_jobs_queued total_jobs_building total_workers registered
for line in req.response.split('\n'):
parts = [x.strip() for x in line.split('\t')]
# parts[0] - function name
if not parts or parts[0] == '.':
continue
if not parts[0].startswith('build:'):
continue
function = parts[0][len('build:'):]
# get total jobs in queue, including the ones being built
try:
queued = int(parts[1])
except:
queued = 0
if ':' in function:
fparts = function.split(':')
job = fparts[-2]
worker = fparts[-1]
workers = job_worker_map.get(job, [])
workers.append(worker)
job_worker_map[job] = worker
# if there are queued tasks, add to demand
if queued > 0:
needed_workers[worker] = needed_workers.get(worker,
0) + queued
elif queued > 0:
# job not specified
job = function
unespecified_jobs[job] = unspecified_jobs.get(job,
0) + queued
# send demand of workers
for job, queued in unspecified_jobs.items():
workers = job_worker_map.get(job)
if not workers:
continue
worker = workers[0]
needed_workers[worker] = needed_workers.get(worker, 0) + queued
return needed_workers
def sendAdminRequest(self):
for connection in self.active_connections:
try:
req = gear.StatusAdminRequest()
connection.sendAdminRequest(req)
except Exception:
print("Exception while listing functions")
return
print("Connection: %s" % (str(connection)))
for line in req.response.split('\n'):
print(" %s" % (line))
def test_readPacket_admin_mix(self):
p1 = gear.Packet(gear.constants.REQ, gear.constants.WORK_COMPLETE,
b'H:127.0.0.1:11\x00' + (b'x' * 5000))
response = b'test\t0\t0\t1\n.\n'
p2 = gear.Packet(gear.constants.REQ, gear.constants.WORK_COMPLETE,
b'H:127.0.0.1:11\x00' + (b'x' * 5000))
req = gear.StatusAdminRequest()
self.conn.admin_requests.append(req)
self.socket._set_data([p1.toBinary() + response + p2.toBinary()])
r1 = self.conn.readPacket()
self.assertEquals(r1, p1)
ra = self.conn.readPacket()
self.assertEqual(ra.response, response)
r2 = self.conn.readPacket()
self.assertEquals(r2, p2)
self.assertEndOfData()
def waitForGearmanToSettle(self):
# If we're running the internal gearman server, it's possible
# that after a restart or reload, we may be immediately ready
# to run jobs but all the gearman workers may not have
# registered yet. Give them a sporting chance to show up
# before we start declaring jobs lost because we don't have
# gearman functions registered for them.
# Spend up to 30 seconds after we connect to the gearman
# server waiting for the set of defined jobs to become
# consistent over a sliding 5 second window.
self.log.info("Waiting for connection to internal Gearman server")
self.waitForServer()
self.log.info("Waiting for gearman function set to settle")
start = time.time()
last_change = start
all_functions = set()
while time.time() - start < 30:
now = time.time()
last_functions = set()
for connection in self.active_connections:
try:
req = gear.StatusAdminRequest()
connection.sendAdminRequest(req)
except Exception:
self.log.exception("Exception while checking functions")
continue
for line in req.response.split('\n'):
parts = [x.strip() for x in line.split()]
if not parts or parts[0] == '.':
continue
last_functions.add(parts[0])
if last_functions != all_functions:
last_change = now
all_functions.update(last_functions)
else:
if now - last_change > 5:
self.log.info("Gearman function set has settled")
break
time.sleep(1)
self.log.info("Done waiting for Gearman server")
def getFunctions(self):
functions = {}
for connection in self.worker.active_connections:
try:
req = gear.StatusAdminRequest()
connection.sendAdminRequest(req, timeout=300)
except Exception:
self.log.exception("Exception while listing functions")
self.worker._lostConnection(connection)
continue
for line in req.response.decode('utf8').split('\n'):
parts = [x.strip() for x in line.split('\t')]
if len(parts) < 4:
continue
# parts[0] - function name
# parts[1] - total jobs queued (including building)
# parts[2] - jobs building
# parts[3] - workers registered
data = functions.setdefault(parts[0], [0, 0, 0])
for i in range(3):
data[i] += int(parts[i + 1])
return functions
def get_queued_image_jobs(self):
'Count the number of image-build and upload jobs queued.'
queued = 0
for connection in self.active_connections:
try:
req = gear.StatusAdminRequest()
connection.sendAdminRequest(req)
except Exception:
self.__log.exception("Exception while listing functions")
self._lostConnection(connection)
continue
for line in req.response.split('\n'):
parts = [x.strip() for x in line.split('\t')]
# parts[0] - function name
# parts[1] - total jobs queued (including building)
# parts[2] - jobs building
# parts[3] - workers registered
if not parts or parts[0] == '.':
continue
if (not parts[0].startswith('image-build:')
and not parts[0].startswith('image-upload:')):
continue
queued += int(parts[1])
return queued
def test_full_packet(self):
req = gear.StatusAdminRequest()
ret = req.isComplete(self.response)
self.assertTrue(ret[0])
self.assertEqual(ret[1], self.remainder)
